pax_global_header00006660000000000000000000000064143701404040014507gustar00rootroot0000000000000052 comment=924aad748242c545ce42c79670dadafd89240d1a pyutil-3.3.2/000077500000000000000000000000001437014040400130425ustar00rootroot00000000000000pyutil-3.3.2/.gitattributes000066400000000000000000000000401437014040400157270ustar00rootroot00000000000000pyutil/_version.py export-subst pyutil-3.3.2/.gitignore000066400000000000000000000004731437014040400150360ustar00rootroot00000000000000*.py[cod] # C extensions *.so # Packages *.egg *.egg-info dist build eggs parts bin var sdist develop-eggs .installed.cfg lib lib64 __pycache__ # Installer logs pip-log.txt # Unit test / coverage reports .coverage .tox nosetests.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject pyutil-3.3.2/.travis.yml000066400000000000000000000003501437014040400151510ustar00rootroot00000000000000language: python cache: pip python: - "2.7" - "3.5" - "3.6" - "3.7-dev" install: - pip install twisted setuptools_trial simplejson script: - python ./setup.py trial -s pyutil.test.current - python setup.py install pyutil-3.3.2/COPYING.GPL000066400000000000000000000573221437014040400145270ustar00rootroot00000000000000This work also comes with the added permission that you may combine it with a work licensed under the OpenSSL license (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the OpenSSL license. This work also comes with the added permission that you may combine it with a work licensed under the Eclipse Public Licence (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Eclipse Public Licence. This work also comes with the added permission that you may combine it with a work licensed under the Q Public Licence (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Q Public Licence. This work also comes with the added permission that you may combine it with a work licensed under the Apache Licence (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Apache Licence. This work also comes with the added permission that you may combine it with a work licensed under the GNU Lesser General Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the GNU Lesser General Public License. This work also comes with the added permission that you may combine it with a work licensed under the Zope Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Zope Public License. This work also comes with the added permission that you may combine it with a work licensed under the Python Software Foundation License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Python Software Foundation License. This work also comes with the added permission that you may combine it with a work licensed under the Academic Free License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Academic Free License. This work also comes with the added permission that you may combine it with a work licensed under the Apple Public Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Apple Public Source License. This work also comes with the added permission that you may combine it with a work licensed under the BitTorrent Open Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the BitTorrent Open Source License. This work also comes with the added permission that you may combine it with a work licensed under the Lucent Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Lucent Public License. This work also comes with the added permission that you may combine it with a work licensed under the Jabber Open Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Jabber Open Source License. This work also comes with the added permission that you may combine it with a work licensed under the Common Development and Distribution License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Common Development and Distribution License. This work also comes with the added permission that you may combine it with a work licensed under the Microsoft Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Microsoft Public License. This work also comes with the added permission that you may combine it with a work licensed under the Microsoft Reciprocal License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Microsoft Reciprocal License. This work also comes with the added permission that you may combine it with a work licensed under the Sun Industry Standards Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Sun Industry Standards Source License. This work also comes with the added permission that you may combine it with a work licensed under the Open Software License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Open Software License. GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. pyutil-3.3.2/COPYING.SPL.txt000066400000000000000000000003051437014040400153460ustar00rootroot00000000000000Permission is hereby granted to any person obtaining a copy of this work to deal in this work without restriction (including the rights to use, modify, distribute, sublicense, and/or sell copies). pyutil-3.3.2/COPYING.TGPPL.rst000066400000000000000000000415421437014040400155770ustar00rootroot00000000000000.. -*- coding: utf-8; fill-column: 77 -*- This work also comes with the added permission that you may combine it with a work licensed under the OpenSSL license (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the OpenSSL license. This work also comes with the added permission that you may combine it with a work licensed under the Eclipse Public Licence (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Eclipse Public Licence. This work also comes with the added permission that you may combine it with a work licensed under the Q Public Licence (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Q Public Licence. This work also comes with the added permission that you may combine it with a work licensed under the Apache Licence (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Apache Licence. This work also comes with the added permission that you may combine it with a work licensed under the GNU Lesser General Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the GNU Lesser General Public License. This work also comes with the added permission that you may combine it with a work licensed under the Zope Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Zope Public License. This work also comes with the added permission that you may combine it with a work licensed under the Python Software Foundation License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Python Software Foundation License. This work also comes with the added permission that you may combine it with a work licensed under the Academic Free License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Academic Free License. This work also comes with the added permission that you may combine it with a work licensed under the Apple Public Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Apple Public Source License. This work also comes with the added permission that you may combine it with a work licensed under the BitTorrent Open Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the BitTorrent Open Source License. This work also comes with the added permission that you may combine it with a work licensed under the Lucent Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Lucent Public License. This work also comes with the added permission that you may combine it with a work licensed under the Jabber Open Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Jabber Open Source License. This work also comes with the added permission that you may combine it with a work licensed under the Common Development and Distribution License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Common Development and Distribution License. This work also comes with the added permission that you may combine it with a work licensed under the Microsoft Public License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Microsoft Public License. This work also comes with the added permission that you may combine it with a work licensed under the Microsoft Reciprocal License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Microsoft Reciprocal License. This work also comes with the added permission that you may combine it with a work licensed under the Sun Industry Standards Source License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Sun Industry Standards Source License. This work also comes with the added permission that you may combine it with a work licensed under the Open Software License (any version) and distribute the resulting combined work, as long as you follow the requirements of the licences of this work in regard to all of the resulting combined work aside from the work licensed under the Open Software License. Transitive Grace Period Public Licence ("TGPPL") v. 1.0 ======================================================= This Transitive Grace Period Public Licence (the "License") applies to any original work of authorship (the "Original Work") whose owner (the "Licensor") has placed the following licensing notice adjacent to the copyright notice for the Original Work: *Licensed under the Transitive Grace Period Public Licence version 1.0* 1. **Grant of Copyright License.** Licensor grants You a worldwide, royalty-free, non-exclusive, sublicensable license, for the duration of the copyright, to do the following: a. to reproduce the Original Work in copies, either alone or as part of a collective work; b. to translate, adapt, alter, transform, modify, or arrange the Original Work, thereby creating derivative works ("Derivative Works") based upon the Original Work; c. to distribute or communicate copies of the Original Work and Derivative Works to the public, with the proviso that copies of Original Work or Derivative Works that You distribute or communicate shall be licensed under this Transitive Grace Period Public Licence no later than 12 months after You distributed or communicated said copies; d. to perform the Original Work publicly; and e. to display the Original Work publicly. 2. **Grant of Patent License.** Licensor grants You a worldwide, royalty-free, non-exclusive, sublicensable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, for the duration of the patents, to make, use, sell, offer for sale, have made, and import the Original Work and Derivative Works. 3. **Grant of Source Code License.** The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work. 4. **Exclusions From License Grant.** Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior permission of the Licensor. Except as expressly stated herein, nothing in this License grants any license to Licensor's trademarks, copyrights, patents, trade secrets or any other intellectual property. No patent license is granted to make, use, sell, offer for sale, have made, or import embodiments of any patent claims other than the licensed claims defined in Section 2. No license is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under terms different from this License any Original Work that Licensor otherwise would have a right to license. 5. **External Deployment.** The term "External Deployment" means the use, distribution, or communication of the Original Work or Derivative Works in any way such that the Original Work or Derivative Works may be used by anyone other than You, whether those works are distributed or communicated to those persons or made available as an application intended for use over a network. As an express condition for the grants of license hereunder, You must treat any External Deployment by You of the Original Work or a Derivative Work as a distribution under section 1(c). 6. **Attribution Rights.** You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent, or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. 7. **Warranty of Provenance and Disclaimer of Warranty.** Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately preceding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of non-infringement, merchantability or fitness for a particular purpose. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to the Original Work is granted by this License except under this disclaimer. 8. **Limitation of Liability.** Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to anyone for any indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to the extent applicable law prohibits such limitation. 9. **Acceptance and Termination.** If, at any time, You expressly assented to this License, that assent indicates your clear and irrevocable acceptance of this License and all of its terms and conditions. If You distribute or communicate copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. This License conditions your rights to undertake the activities listed in Section 1, including your right to create Derivative Works based upon the Original Work, and doing so without honoring these terms and conditions is prohibited by copyright law and international treaty. Nothing in this License is intended to affect copyright exceptions and limitations (including 'fair use' or 'fair dealing'). This License shall terminate immediately and You may no longer exercise any of the rights granted to You by this License upon your failure to honor the conditions in Section 1(c). 10. **Termination for Patent Action.** This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. 11. **Jurisdiction, Venue and Governing Law.** Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of copyright or patent law in the appropriate jurisdiction. This section shall survive the termination of this License. 12. **Attorneys' Fees.** In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. 13. **Miscellaneous.** If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. 14. **Definition of "You" in This License.** "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 15. **Right to Use.** You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. 16. **Modification of This License.** This License is Copyright © 2007 Zooko Wilcox-O'Hearn. Permission is granted to copy, distribute, or communicate this License without modification. Nothing in this License permits You to modify this License as applied to the Original Work or to Derivative Works. However, You may modify the text of this License and copy, distribute or communicate your modified version (the "Modified License") and apply it to other original works of authorship subject to the following conditions: (i) You may not indicate in any way that your Modified License is the "Transitive Grace Period Public Licence" or "TGPPL" and you may not use those names in the name of your Modified License; and (ii) You must replace the notice specified in the first paragraph above with the notice "Licensed under " or with a notice of your own that is not confusingly similar to the notice in this License. pyutil-3.3.2/CREDITS000066400000000000000000000017371437014040400140720ustar00rootroot00000000000000 This is at least a partial credits-file of people that have contributed to the pyutil project. It is formatted to allow easy grepping and beautification by scripts. The fields are: name (N), email (E), web-address (W), PGP key ID and fingerprint (P), description (D), and snail-mail address (S). Thanks, Zooko ---------- N: The Evil Geniuses For A Better Tomorrow D: turned some crazy ideas into a crazy reality N: zooko E: zookog@gmail.com D: everything up to version 2.0.0 N: Lars the Talking Alien E: lars@mad-scientist.com D: came to us in dreams and made us become Enlightened Ones S: 644 Church St., Mountain View, 94041 CA, USA, 3 Sol, Milky Way, above the fireplace N: Faried Nawaz E: fn@hungry.org D: the c_xor module N: Lele Gaifax E: lele@nautilus.homeip.net D: patch to fileutil -- close tempfile fd N: Nathan Wilcox E: nejucomo+github@gmail.com D: improved tests and some documentation N: George Hopkins D: Python 3 support N: Dan Callaghan D: Python 3 support pyutil-3.3.2/MANIFEST.in000066400000000000000000000002341437014040400145770ustar00rootroot00000000000000include COPYING.GPL COPYING.TGPPL.rst COPYING.SPL.txt CREDITS README.rst include MANIFEST.in include versioneer.py graft pyutil include pyutil/_version.py pyutil-3.3.2/README.rst000066400000000000000000000166721437014040400145450ustar00rootroot00000000000000.. -*- coding: utf-8; fill-column: 77 -*- pyutil -- a library of useful Python functions and classes ========================================================== Many of these utilities (or their ancestors) were developed originally by Zooko Wilcox-O'Hearn for the Mojo Nation, Mnet, Allmydata.com "Mountain View", Tahoe-LAFS, or SimpleGeo's products. Please note that version 2.X is based on Python 2. Version 3.X will has support for Python 3. .. image:: https://travis-ci.org/tpltnt/pyutil.svg?branch=master :target: https://travis-ci.org/tpltnt/pyutil utilities ========= current ------- - mathutil.py_ - integer power, floor, ceil, and nearest multiples; permute and fit slope - memutil.py_ - statistics and diagnostics for memory use and garbage collection - platformutil.py_ - get platform including Linux distro; more accurate and less noisy than platform.platform() - strutil.py_ - common prefix and suffix of two strings, and newline processing - assertutil.py_ - test preconditions, postconditions, and assertions - benchutil.py_ - benchmark a function by running it repeatedly - fileutil.py_ - work with files and directories - iputil.py_ - query available local IPv4 addresses - jsonutil.py_ - wrapper around simplejson which converts decimal inputs to Python Decimal objects instead of to Python floats - lineutil.py_ - remove extra whitespace from files - testutil.py_ - utilities for use in unit tests, especially in Twisted - time_format.py_ - date and time formatting operations - version_class.py_ - parse version strings into a Version Number object - verlib.py_ - utility to compare version strings, by Tarek Ziadé out of shape ------------ I don't currently use these, but I still think they are possibly good ideas. - nummedobj.py_ - number objects in order of creation for consistent debug output - observer.py_ - the Observer pattern - increasing.py_ - an implementation of a monotonically-increasing timer; By the way a future, better implementation of this would use CLOCK_MONOTONIC or CLOCK_MONOTONIC_RAW if it were available: http://stackoverflow.com/questions/1205722/how-do-i-get-monotonic-time-durations-in-python/1205762#1205762 - repeatable_random.py_ - Make the random and time modules deterministic, so that executions can be reproducible. - strutil.py_ - string utilities - cache.py_ - multiple implementations of a least-recently-used in-memory caching strategy, optimized for different sizes (note: I, Zooko, nowadays prefer a random-replacement cache eviction strategy over least-recently-used because the former has more consistent and predictable behavior) - odict.py_ - ordered dictionary implementation: see PEP 372. Note: there is now (as of Python 2.7) an ordered dict implementation in the standard library, but I haven't checked if it is as good as this one. - zlibutil.py_ - zlib decompression in limited memory deprecated ---------- I no longer use these and I don't recommend that you do either. - logutil.py_ - send log messages to Twisted logger if present, else Python library logger - weakutil.py_ - allows a bound method's object to be GC'd - twistedutil.py_ - callLater_weakly, a variant of Twisted's callLater which interacts more nicely with weakrefs - PickleSaver.py_ - make all or part of an object persistent, by saving it to disk when it's garbage collected - humanreadable.py_ - an improved version of the builtin repr() function - find_exe.py_ - try different paths in search of an executable - dictutil.py_ - several specialized dict extensions, as well as some convenient functions for working with dicts - randutil.py_ - various ways to get random bytes - xor.py_ - xor two same-length strings together Thanks to Peter Westlake and Ravi Pinjala for help documenting what these do. download ======== https://pypi.python.org/pypi/pyutil source code (git) ================= https://github.com/tpltnt/pyutil issue tracker ============= https://github.com/tpltnt/pyutil/issues tests and benchmarks ==================== Testing requires these dependencies: ``twisted``, ``setuptools_trial``, and ``simplejson``. You can install them by running: ``pip install twisted setuptools_trial simplejson``. To run tests: ``python ./setup.py trial -s pyutil.test.current``. You can also run the tests with the standard pyunit test runner instead of trial, but a couple of the tests will fail due to the absence of Trial's "Skip This Test" feature. You can also run the tests of the out-of-shape and deprecated modules: ``python ./setup.py trial -s pyutil.test.out_of_shape`` ``python ./setup.py trial -s pyutil.test.deprecated`` Or of all modules: ``python ./setup.py trial -s pyutil.test`` Some modules have self-benchmarks provided. For example, to benchmark the cache module: ``python -OOu -c 'from pyutil.test import test_cache; test_cache.quick_bench()'`` or for more complete and time-consuming results: ``python -OOu -c 'from pyutil.test import test_cache; test_cache.slow_bench()'`` (The "-O" is important when benchmarking, since cache has extensive self-tests that are optimized out when -O is included.) testing with virtualenv ----------------------- You can install all testing tools and test-time dependencies in a local `virtualenv` which avoids interference with user- or system-wide python packages. Here's an example of this process (command output omitted): .. code:: bash $ git clone 'https://github.com/tpltnt/pyutil' $ cd pyutil/ $ virtualenv ./venv $ ./venv/bin/pip install twisted setuptools_trial simplejson $ ./venv/bin/python ./setup.py trial -s pyutil.test LICENCE ======= You may use this package under the GNU General Public License, version 2 or, at your option, any later version. You may use this package under the Transitive Grace Period Public Licence, version 1.0, or at your option, any later version. (You may choose to use this package under the terms of either licence, at your option.) You may use this package under the Simple Permissive Licence, version 1 or, at your option, any later version. See the file COPYING.GPL_ for the terms of the GNU General Public License, version 2. See the file COPYING.TGPPL.rst_ for the terms of the Transitive Grace Period Public Licence, version 1.0. See the file COPYING.SPL.txt_ for the terms of the Simple Permissive Licence, version 1. .. _COPYING.GPL: COPYING.GPL .. _COPYING.TGPPL.rst: COPYING.TGPPL.rst .. _COPYING.SPL.txt: COPYING.SPL.txt .. _mathutil.py: pyutil/mathutil.py .. _memutil.py: pyutil/memutil.py .. _platformutil.py: pyutil/platformutil.py .. _strutil.py: pyutil/strutil.py .. _assertutil.py: pyutil/assertutil.py .. _benchutil.py: pyutil/benchutil.py .. _fileutil.py: pyutil/fileutil.py .. _iputil.py: pyutil/iputil.py .. _jsonutil.py: pyutil/jsonutil.py .. _lineutil.py: pyutil/lineutil.py .. _testutil.py: pyutil/testutil.py .. _time_format.py: pyutil/time_format.py .. _version_class.py: pyutil/version_class.py .. _zlibutil.py: pyutil/zlibutil.py .. _nummedobj.py: pyutil/nummedobj.py .. _observer.py: pyutil/observer.py .. _increasing.py: pyutil/increasing.py .. _repeatable_random.py: pyutil/repeatable_random.py .. _cache.py: pyutil/cache.py .. _odict.py: pyutil/odict.py .. _logutil.py: pyutil/logutil.py .. _weakutil.py: pyutil/weakutil.py .. _twistedutil.py: pyutil/twistedutil.py .. _PickleSaver.py: pyutil/PickleSaver.py .. _humanreadable.py: pyutil/humanreadable.py .. _find_exe.py: pyutil/find_exe.py .. _dictutil.py: pyutil/dictutil.py .. _randutil.py: pyutil/randutil.py .. _xor.py: pyutil/xor/xor.py .. _verlib.py: pyutil/verlib.py pyutil-3.3.2/ez_setup.py000066400000000000000000000206131437014040400152540ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- """Bootstrap setuptools installation If you want to use setuptools in your package's setup.py, just include this file in the same directory with it, and add this to the top of your setup.py:: from ez_setup import use_setuptools use_setuptools() If you want to require a specific version of setuptools, set a download mirror, or use an alternate download directory, you can do so by supplying the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import os, sys DEFAULT_VERSION = "0.6c7" DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] md5_data = { 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2', 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e', 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372', } def _validate_md5(egg_name, data): if egg_name in md5_data: from md5 import md5 digest = md5(data).hexdigest() if digest != md5_data[egg_name]: print >>sys.stderr, ( "md5 validation of %s failed! (Possible download problem?)" % egg_name ) sys.exit(2) return data # The following code to parse versions is copied from pkg_resources.py so that # we can parse versions without importing that module. import re component_re = re.compile(r'(\d+ | [a-z]+ | \.| -)', re.VERBOSE) replace = {'pre':'c', 'preview':'c','-':'final-','rc':'c','dev':'@'}.get def _parse_version_parts(s): for part in component_re.split(s): part = replace(part,part) if not part or part=='.': continue if part[:1] in '0123456789': yield part.zfill(8) # pad for numeric comparison else: yield '*'+part yield '*final' # ensure that alpha/beta/candidate are before final def parse_version(s): parts = [] for part in _parse_version_parts(s.lower()): if part.startswith('*'): if part<'*final': # remove '-' before a prerelease tag while parts and parts[-1]=='*final-': parts.pop() # remove trailing zeros from each series of numeric parts while parts and parts[-1]=='00000000': parts.pop() parts.append(part) return tuple(parts) def setuptools_is_new_enough(required_version): """Return True if setuptools is already installed and has a version number >= required_version.""" if 'pkg_resources' in sys.modules: import pkg_resources try: pkg_resources.require('setuptools >= %s' % (required_version,)) except pkg_resources.VersionConflict: # An insufficiently new version is installed. return False else: return True else: try: import pkg_resources except ImportError: # Okay it is not installed. return False else: try: pkg_resources.require('setuptools >= %s' % (required_version,)) except pkg_resources.VersionConflict: # An insufficiently new version is installed. pkg_resources.__dict__.clear() # "If you want to be absolutely sure... before deleting it." --said PJE on IRC del sys.modules['pkg_resources'] return False else: pkg_resources.__dict__.clear() # "If you want to be absolutely sure... before deleting it." --said PJE on IRC del sys.modules['pkg_resources'] return True def use_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, min_version=None, download_delay=15 ): """Automatically find/download setuptools and make it available on sys.path `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where setuptools will be downloaded, if it is not already available. If `download_delay` is specified, it should be the number of seconds that will be paused before initiating a download, should one be required. If an older version of setuptools is installed, this routine will print a message to ``sys.stderr`` and raise SystemExit in an attempt to abort the calling script. """ if min_version is None: min_version = version if not setuptools_is_new_enough(min_version): egg = download_setuptools(version, min_version, download_base, to_dir, download_delay) sys.path.insert(0, egg) import setuptools; setuptools.bootstrap_install_from = egg def download_setuptools( version=DEFAULT_VERSION, min_version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay = 15 ): """Download setuptools from a specified location and return its filename `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. """ import urllib2, shutil egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3]) url = download_base + egg_name saveto = os.path.join(to_dir, egg_name) src = dst = None if not os.path.exists(saveto): # Avoid repeated downloads try: from distutils import log if delay: log.warn(""" --------------------------------------------------------------------------- This script requires setuptools version >= %s to run (even to display help). I will attempt to download setuptools for you (from %s), but you may need to enable firewall access for this script first. I will start the download in %d seconds. (Note: if this machine does not have network access, please obtain the file %s and place it in this directory before rerunning this script.) ---------------------------------------------------------------------------""", min_version, download_base, delay, url ); from time import sleep; sleep(delay) log.warn("Downloading %s", url) src = urllib2.urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = _validate_md5(egg_name, src.read()) dst = open(saveto,"wb"); dst.write(data) finally: if src: src.close() if dst: dst.close() return os.path.realpath(saveto) def main(argv, version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" if setuptools_is_new_enough(version): if argv: from setuptools.command.easy_install import main main(argv) else: print "Setuptools version",version,"or greater has been installed." print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' else: egg = None try: egg = download_setuptools(version, min_version=version, delay=0) sys.path.insert(0,egg) from setuptools.command.easy_install import main return main(list(argv)+[egg]) # we're done here finally: if egg and os.path.exists(egg): os.unlink(egg) def update_md5(filenames): """Update our built-in md5 registry""" import re from md5 import md5 for name in filenames: base = os.path.basename(name) f = open(name,'rb') md5_data[base] = md5(f.read()).hexdigest() f.close() data = [" %r: %r,\n" % it for it in md5_data.items()] data.sort() repl = "".join(data) import inspect srcfile = inspect.getsourcefile(sys.modules[__name__]) f = open(srcfile, 'rb'); src = f.read(); f.close() match = re.search("\nmd5_data = {\n([^}]+)}", src) if not match: print >>sys.stderr, "Internal error!" sys.exit(2) src = src[:match.start(1)] + repl + src[match.end(1):] f = open(srcfile,'w') f.write(src) f.close() if __name__=='__main__': if '--md5update' in sys.argv: sys.argv.remove('--md5update') update_md5(sys.argv[1:]) else: main(sys.argv[1:]) pyutil-3.3.2/pyutil/000077500000000000000000000000001437014040400143705ustar00rootroot00000000000000pyutil-3.3.2/pyutil/PickleSaver.py000066400000000000000000000214411437014040400171540ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ An object that makes some of the attributes of your class persistent, pickling them and lazily writing them to a file. """ # from the Python Standard Library import os try: import cPickle as pickle # PY2 except ImportError: import pickle # PY3 import warnings # from the pyutil library from . import fileutil from . import nummedobj from . import twistedutil # from the Twisted library from twisted.python import log class PickleSaver(nummedobj.NummedObj): """ This makes some of the attributes of your class persistent, saving them in a pickle and saving them lazily. The general idea: You are going to tell PickleSaver which of your attributes ought to be persistently saved, and the name of a file to save them in. Those attributes will get saved to disk, and when your object is instantiated those attributes will get set to the values loaded from the file. Usage: inherit from PickleSaver and call PickleSaver.__init__() in your constructor. You will pass arguments to PickleSaver.__init__() telling it which attributes to save, which file to save them in, and what values they should have if there is no value stored for them in the file. Note: do *not* assign values to your persistent attributes in your constructor, because you might thus overwrite their persistent values. Then whenever you change one of the persistent attributes, call self.lazy_save() (it won't *really* save -- it'll just schedule a save for DELAY minutes later.) If you update an attribute and forget to call self.lazy_save() then the change will not be saved, unless you later call self.lazy_save() before you shut down. Data could be lost if the Python interpreter were to die unexpectedly (for example, due to a segfault in a compiled machine code module or due to the Python process being killed without warning via SIGKILL) before the delay passes. However if the Python interpreter shuts down cleanly (i.e., if it garbage collects and invokes the __del__ methods of the collected objects), then the data will be saved at that time (unless your class has the "not-collectable" problem: http://python.org/doc/current/lib/module-gc.html -- search in text for "uncollectable"). Note: you can pass DELAY=0 to make PickleSaver a not-so-lazy saver. The advantage of laziness is that you don't touch the disk as often -- touching disk is a performance cost. To cleanly shutdown, invoke shutdown(). Further operations after that will result in exceptions. """ class ExtRes: """ This is for holding things (external resources) that PickleSaver needs to finalize after PickleSaver is killed. (post-mortem finalization) In particular, this holds the names and values of all attributes that have been changed, so that after the PickleSaver is garbage-collected those values will be saved to the persistent file. """ def __init__(self, fname, objname): self.fname = fname self.objname = objname self.dirty = False # True iff the attrs have been changed and need to be saved to disk; When you change this flag from False to True, you schedule a save task for 10 minutes later. When the save task goes off it changes the flag from True to False. self.savertask = None self.valstr = None # the pickled (serialized, string) contents of the attributes that should be saved def _save_to_disk(self): if self.valstr is not None: log.msg("%s._save_to_disk(): fname: %s" % (self.objname, self.fname,)) of = open(self.fname + ".tmp", "wb") of.write(self.valstr) of.flush() of.close() of = None fileutil.remove_if_possible(self.fname) fileutil.rename(self.fname + ".tmp", self.fname) log.msg("%s._save_to_disk(): now, having finished write(), os.path.isfile(%s): %s" % (self, self.fname, os.path.isfile(self.fname),)) self.valstr = None self.dirty = False try: self.savertask.callId.cancel() except: pass self.savertask = None def shutdown(self): if self.dirty: self._save_to_disk() if self.savertask: try: self.savertask.callId.cancel() except: pass self.savertask = None def __del__(self): self.shutdown() def __init__(self, fname, attrs, DELAY=60*60, savecb=None): """ @param attrs: a dict whose keys are the names of all the attributes to be persistently stored and whose values are the initial default value that the attribute gets set to the first time it is ever used; After this first initialization, the value will be persistent so the initial default value will never be used again. @param savecb: if not None, then it is a callable that will be called after each save completes (useful for unit tests) (savecb doesn't get called after a shutdown-save, only after a scheduled save) """ warnings.warn("deprecated", DeprecationWarning) nummedobj.NummedObj.__init__(self) self._DELAY = DELAY self._attrnames = attrs.keys() self._extres = PickleSaver.ExtRes(fname=fname, objname=self.__repr__()) self._savecb = savecb for attrname, defaultval in attrs.items(): setattr(self, attrname, defaultval) try: attrdict = pickle.loads(open(self._extres.fname, "rb").read()) for attrname, attrval in attrdict.items(): if not hasattr(self, attrname): log.msg("WARNING: %s has no attribute named %s on load from disk, value: %s." % (self, attrname, attrval,)) setattr(self, attrname, attrval) except (pickle.UnpicklingError, IOError, EOFError,) as le: try: attrdict = pickle.loads(open(self._extres.fname + ".tmp", "rb").read()) for attrname, attrval in attrdict.items(): if not hasattr(self, attrname): log.msg("WARNING: %s has no attribute named %s on load from disk, value: %s." % (self, attrname, attrval,)) setattr(self, attrname, attrval) except (pickle.UnpicklingError, IOError, EOFError,) as le2: log.msg("Got exception attempting to load attrs. (This is normal if this is the first time you've used this persistent %s object.) fname: %s, le: %s, le2: %s" % (self.__class__, self._extres.fname, le, le2,)) self.lazy_save() def _store_attrs_in_extres(self): d = {} for attrname in self._attrnames: d[attrname] = getattr(self, attrname) # log.msg("%s._store_attrs_in_extres: attrname: %s, val: %s" % (self, attrname, getattr(self, attrname),)) # pickle the attrs now, to ensure that there are no reference cycles self._extres.valstr = pickle.dumps(d, True) # log.msg("%s._store_attrs_in_extres: valstr: %s" % (self, self._extres.valstr,)) self._extres.dirty = True def _save_to_disk(self): log.msg("%s._save_to_disk()" % (self,)) self._extres._save_to_disk() if self._savecb: self._savecb() def _lazy_save(self, delay=None): """ @deprecated: use lazy_save() instead """ return self.lazy_save(delay) def lazy_save(self, delay=None): """ @param delay: how long from now before the data gets saved to disk, or `None' in order to use the default value provided in the constructor """ if delay is None: delay=self._DELAY # copy the values into extres so that if `self' gets garbage-collected the values will be written to disk during post-mortem finalization. (This also marks it as dirty.) self._store_attrs_in_extres() newsavetask = twistedutil.callLater_weakly(delay, self._save_to_disk) if self._extres.savertask: if self._extres.savertask.callId.getTime() < newsavetask.callId.getTime(): try: newsavetask.callId.cancel() except: pass else: try: self._extres.savertask.callId.cancel() except: pass self._extres.savertask = newsavetask else: self._extres.savertask = newsavetask def shutdown(self): self.extres.shutdown() self.extres = None pyutil-3.3.2/pyutil/__init__.py000066400000000000000000000003121437014040400164750ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- """ Library of useful Python functions and classes. """ from . import _version __version__ = _version.get_versions()['version'] pyutil-3.3.2/pyutil/_version.py000066400000000000000000000561621437014040400166000ustar00rootroot00000000000000 # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. # Generated by versioneer-0.28 # https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" import errno import os import re import subprocess import sys from typing import Callable, Dict import functools def get_keywords(): """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = " (tag: 3.3.2)" git_full = "924aad748242c545ce42c79670dadafd89240d1a" git_date = "2023-02-06 09:57:40 +0100" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: """Container for Versioneer configuration parameters.""" def get_config(): """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "pep440" cfg.tag_prefix = "" cfg.parentdir_prefix = "pyutil-" cfg.versionfile_source = "pyutil/_version.py" cfg.verbose = False return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue if verbose: print("unable to run %s" % dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) return None, process.returncode return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def plus_or_dot(pieces): """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces): """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver): """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces): """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%d" % (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%d" % pieces["distance"] return rendered def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%s" % pieces["short"] return rendered def render_pep440_post_branch(pieces): """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces): """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces): """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} def get_versions(): """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which # case we can only use expanded keywords. cfg = get_config() verbose = cfg.verbose try: return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass try: root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to find root of source tree", "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) return render(pieces, cfg.style) except NotThisMethod: pass try: if cfg.parentdir_prefix: return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) except NotThisMethod: pass return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} pyutil-3.3.2/pyutil/assertutil.py000066400000000000000000000022771437014040400171510ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ Tests useful in assertion checking, prints out nicely formated messages too. """ from .humanreadable import hr def _format_error(prefix, args, kwargs): if prefix: msgbuf=[prefix] if args or kwargs: msgbuf.append(": ") else: msgbuf=[] if args: msgbuf.append(", ".join(["%s %s" % tuple(map(hr, (arg, type(arg),))) for arg in args])) if kwargs: if args: msgbuf.append(", ") msgbuf.append(", ".join(["%s: %s %s" % tuple(map(hr, (k, kwargs[k], type(kwargs[k]),))) for k in sorted(kwargs.keys())])) return "".join(msgbuf) def _assert(___cond=False, *args, **kwargs): if ___cond: return True raise AssertionError(_format_error(None, args, kwargs)) def precondition(___cond=False, *args, **kwargs): if ___cond: return True raise AssertionError(_format_error("precondition", args, kwargs)) def postcondition(___cond=False, *args, **kwargs): if ___cond: return True raise AssertionError(_format_error("postcondition", args, kwargs)) pyutil-3.3.2/pyutil/benchutil.py000066400000000000000000000263741437014040400167330ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ Benchmark a function for its behavior with respect to N. How to use this module: 1. Define a function which runs the code that you want to benchmark. The function takes a single argument which is the size of the task (i.e. the "N" parameter). Pass this function as the first argument to rep_bench(), and N as the second, e.g.: >>> from pyutil.benchutil import rep_bench >>> def fib(N): ... if N <= 1: ... return 1 ... else: ... return fib(N-1) + fib(N-2) ... >>> rep_bench(fib, 25, UNITS_PER_SECOND=1000) best: 1.968e+00, 3th-best: 1.987e+00, mean: 2.118e+00, 3th-worst: 2.175e+00, worst: 2.503e+00 (of 10) The output is reporting the number of milliseconds that executing the function took, divided by N, from ten different invocations of fib(). It reports the best, worst, M-th best, M-th worst, and mean, where "M" is 1/4 of the number of invocations (in this case 10). 2. Now run it with different values of N and look for patterns: >>> for N in 1, 5, 9, 13, 17, 21: ... print "%2d" % N, ... rep_bench(fib, N, UNITS_PER_SECOND=1000000) ... 1 best: 9.537e-01, 3th-best: 9.537e-01, mean: 1.121e+00, 3th-worst: 1.192e+00, worst: 2.146e+00 (of 10) 5 best: 5.722e-01, 3th-best: 6.199e-01, mean: 7.200e-01, 3th-worst: 8.106e-01, worst: 8.106e-01 (of 10) 9 best: 2.437e+00, 3th-best: 2.464e+00, mean: 2.530e+00, 3th-worst: 2.570e+00, worst: 2.676e+00 (of 10) 13 best: 1.154e+01, 3th-best: 1.168e+01, mean: 5.638e+01, 3th-worst: 1.346e+01, worst: 4.478e+02 (of 10) 17 best: 6.230e+01, 3th-best: 6.247e+01, mean: 6.424e+01, 3th-worst: 6.460e+01, worst: 7.294e+01 (of 10) 21 best: 3.376e+02, 3th-best: 3.391e+02, mean: 3.521e+02, 3th-worst: 3.540e+02, worst: 3.963e+02 (of 10) >>> print_bench_footer(UNITS_PER_SECOND=1000000) all results are in time units per N time units per second: 1000000; seconds per time unit: 0.000001 (The pattern here is that as N grows, the time per N grows.) 2. If you need to do some setting up before the code can run, then put the setting-up code into a separate function so that it won't be included in the timing measurements. A good way to share state between the setting-up function and the main function is to make them be methods of the same object, e.g.: >>> import random >>> class O: ... def __init__(self): ... self.l = [] ... def setup(self, N): ... del self.l[:] ... self.l.extend(range(N)) ... random.shuffle(self.l) ... def sort(self, N): ... self.l.sort() ... >>> o = O() >>> for N in 1000, 10000, 100000, 1000000: ... print "%7d" % N, ... rep_bench(o.sort, N, o.setup) ... 1000 best: 4.830e+02, 3th-best: 4.950e+02, mean: 5.730e+02, 3th-worst: 5.858e+02, worst: 7.451e+02 (of 10) 10000 best: 6.342e+02, 3th-best: 6.367e+02, mean: 6.678e+02, 3th-worst: 6.851e+02, worst: 7.848e+02 (of 10) 100000 best: 8.309e+02, 3th-best: 8.338e+02, mean: 8.435e+02, 3th-worst: 8.540e+02, worst: 8.559e+02 (of 10) 1000000 best: 1.327e+03, 3th-best: 1.339e+03, mean: 1.349e+03, 3th-worst: 1.357e+03, worst: 1.374e+03 (of 10) 3. Useful fact! rep_bench() returns a dict containing the numbers. 4. Things to fix: a. I used to have it hooked up to use the "hotshot" profiler on the code being measured. I recently tried to change it to use the newer cProfile profiler instead, but I don't understand the interface to cProfiler so it just gives an exception if you pass profile=True. Please fix this and send me a patch. xxx change it to statprof b. Wouldn't it be great if this script emitted results in a json format that was understood by a tool to make pretty interactive explorable graphs? The pretty graphs could look like those on http://speed.pypy.org/ . Please make this work and send me a patch! """ from __future__ import print_function import cProfile, operator, time from decimal import Decimal as D import threading as thread #from pyutil import jsonutil as json import platform if 'windows' in platform.system().lower(): clock = time.clock else: clock = time.time from .assertutil import _assert def makeg(func): def blah(n, func=func): for i in xrange(n): func() return blah def to_decimal(x): """ See if D(x) returns something. If instead it raises TypeError, x must have been a float, so convert it to Decimal by way of string. (In Python >= 2.7, D(x) does this automatically. """ try: return D(x) except TypeError: return D("%0.54f" % (x,)) def mult(a, b): """ If we get TypeError from * (possibly because one is float and the other is Decimal), then promote them both to Decimal. """ try: return a * b except TypeError: return to_decimal(a) * to_decimal(b) def _measure_empty_func_with_runtime(profile=False): return _bench_it_until_time(do_nothing, 2**32, runtime=0.03) def _measure_empty_func_with_reps(profile=False): return _bench_it_until_reps(do_nothing, 2**32, runreps=1000) def rep_bench(func, n, runtime=None, initfunc=None, runreps=None, runiters=10, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False): """ @param quiet Don't print anything--just return the results dict. @param runtime How many seconds to run the inner loop (for measuring the time of the code-under-measurement). If None then do it runreps times regardless of how many seconds it takes. @param runreps How many times to run the inner loop (for measuring the time of the code-under-measurement). If None then do it until runtime seconds have passed. @param runiters How many times to run the outer loop (for generating statistics about the distribution of runtimes of the code-under-measurement). """ assert isinstance(n, int), (n, type(n)) assert (runtime is None) != (runreps is None), "Choose either runtime mode or runreps mode. runtime: %s, runreps: %s" % (runtime, runreps,) global worstemptymeasure if runtime is not None: (time_per_rep, reps) = _measure_empty_func_with_runtime(profile=profile) if time_per_rep * MARGINOFERROR >= runtime: raise BadMeasure("Apparently simply invoking an empty Python function can take as long as %0.10f seconds, and we were running reps for only about %0.10f seconds. So the measurement of the runtime of the code under benchmark is not reliable. Please pass a higher number for the 'runtime' argument to rep_bench().") else: (time_per_rep, reps) = _measure_empty_func_with_reps(profile=profile) if (worstemptymeasure is None) or (time_per_rep > worstemptymeasure): worstemptymeasure = time_per_rep tls = [] # (elapsed time per rep in seconds, numreps) while len(tls) < runiters: if initfunc: initfunc(n) if profile: import statprof statprof.start() try: tl, reps = bench_it(func, n, runtime=runtime, runreps=runreps) finally: statprof.stop() else: tl, reps = bench_it(func, n, runtime=runtime, runreps=runreps) tls.append((tl, reps)) sumtls = sum([tl for (tl, reps) in tls]) mean = sumtls / len(tls) tls.sort() worst = tls[-1][0] best = tls[0][0] if best < (worstemptymeasure * MARGINOFERROR): raise BadMeasure("Apparently simply invoking an empty Python function can take as long as %0.10f seconds, and we were running reps for only about %0.10f seconds. So the measurement of the runtime of the code under benchmark is not reliable. Please pass a higher number for the 'runtime' argument to rep_bench().") m = len(tls)//4 if m > 0: mthbest = tls[m-1][0] mthworst = tls[-m][0] else: mthbest = tls[0][0] mthworst = tls[-1][0] # The +/-0 index is the best/worst, the +/-1 index is the 2nd-best/worst, # etc, so we use mp1 to name it. mp1 = m+1 res = { 'worst': mult(worst, UNITS_PER_SECOND)/n, 'best': mult(best, UNITS_PER_SECOND)/n, 'mp1': mp1, 'mth-best': mult(mthbest, UNITS_PER_SECOND)/n, 'mth-worst': mult(mthworst, UNITS_PER_SECOND)/n, 'mean': mult(mean, UNITS_PER_SECOND)/n, 'num': len(tls), } if not quiet: print("best: %(best)#8.03e, %(mp1)3dth-best: %(mth-best)#8.03e, mean: %(mean)#8.03e, %(mp1)3dth-worst: %(mth-worst)#8.03e, worst: %(worst)#8.03e (of %(num)6d)" % res) return res MARGINOFERROR = 10 worstemptymeasure = 0 class BadMeasure(Exception): """ Either the clock wrapped (which happens with time.clock()) or it went backwards (which happens with time.time() on rare occasions), or the code being measured completed too fast to accurately measure. """ def __init__(self, msg): self.msg = msg def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.msg) def __str__(self): return self.__repr__() def do_nothing(n): pass def _interrupt_me_later(delay): time.sleep(delay) thread.interrupt_main() def _bench_it_until_time(func, n, runtime=1.0): reps = 0 st = clock() # We run it once before starting the timer so that we'll have a # resulting measurement, even if interrupted before we finish the # first interruptible run. func(n) reps += 1 sto = clock() try: timer_thread = thread.start_new_thread(_interrupt_me_later, (runtime,)) while True: func(n) reps += 1 sto = clock() except KeyboardInterrupt: timeelapsed = sto - st if timeelapsed <= 0: raise BadMeasure("timeelapsed: %s, reps: %s" % (timeelapsed, reps)) return (timeelapsed / reps, reps) def _bench_it_until_reps(func, n, runreps=1000): st = clock() for i in range(runreps): func(n) sto = clock() timeelapsed = sto - st if timeelapsed <= 0: raise BadMeasure("timeelapsed: %s, runreps: %s" % (timeelapsed, runreps)) return (timeelapsed / runreps, runreps) def bench_it(func, n, runtime=1.0, runreps=None): assert (runtime is None) != (runreps is None), "Choose either runtime mode or runreps mode. runtime: %s, runreps: %s" % (runtime, runreps,) if runtime is not None: return _bench_it_until_time(func, n, runtime=runtime) else: return _bench_it_until_reps(func, n, runreps=runreps) def bench(func, initfunc=None, runtime=1.0, TOPXP=21, profile=False, profresults="pyutil-benchutil.prof", outputjson=False, jsonresultsfname="pyutil-benchutil-results.json", UNITS_PER_SECOND=1): BSIZES = [] for i in range(TOPXP-6, TOPXP+1, 2): n = int(2 ** i) if n < 1: n = 1 if BSIZES and n <= BSIZES[-1]: n *= 2 BSIZES.append(n) res = {} for BSIZE in BSIZES: print("N: %7d," % BSIZE) r = rep_bench(func, BSIZE, initfunc=initfunc, runtime=runtime, profile=profile, profresults=profresults, UNITS_PER_SECOND=UNITS_PER_SECOND) res[BSIZE] = r #if outputjson: # write_file(jsonresultsfname, json.dumps(res)) return res def print_bench_footer(UNITS_PER_SECOND=1): print("all results are in time units per N") print("time units per second: %s; seconds per time unit: %s" % (UNITS_PER_SECOND, D(1)/UNITS_PER_SECOND)) pyutil-3.3.2/pyutil/cache.py000066400000000000000000000655601437014040400160210ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ This module offers three implementations of an LRUCache, which is a dict that drops items according to a Least-Recently-Used policy if the dict exceeds a fixed maximum size. Warning: if -O optimizations are not turned on then LRUCache performs extensive self-analysis in every function call, which can take minutes and minutes for a large cache. Turn on -O, or comment out ``assert self._assert_invariants()`` """ import operator from .assertutil import _assert, precondition from .humanreadable import hr class LRUCache: """ An efficient least-recently-used cache. It keeps an LRU queue, and when the number of items in the cache reaches maxsize, it removes the least recently used item. "Looking" at an item, key, or value such as with "has_key()" makes that item become the most recently used item. You can also use "refresh()" to explicitly make an item become the most recently used item. Adding an item that is already in the dict *does* make it the most- recently-used item although it does not change the state of the dict itself. See also SmallLRUCache (below), which is faster in some cases. """ class ItemIterator: def __init__(self, c): self.c = c self.i = c.d[c.hs][2] def __iter__(self): return self def __next__(self): if self.i is self.c.ts: raise StopIteration() k = self.i precondition(k in self.c.d, "The iterated LRUCache doesn't have the next key. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", k, self.c) (v, p, n,) = self.c.d[k] self.i = n return (k, v,) def next(self): return self.__next__() class KeyIterator: def __init__(self, c): self.c = c self.i = c.d[c.hs][2] def __iter__(self): return self def __next__(self): if self.i is self.c.ts: raise StopIteration() k = self.i precondition(k in self.c.d, "The iterated LRUCache doesn't have the next key. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", k, self.c) (v, p, n,) = self.c.d[k] self.i = n return k def next(self): return self.__next__() class ValIterator: def __init__(self, c): self.c = c self.i = c.d[c.hs][2] def __iter__(self): return self def __next__(self): if self.i is self.c.ts: raise StopIteration() precondition(self.i in self.c.d, "The iterated LRUCache doesn't have the next key. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c) (v, p, n,) = self.c.d[self.i] self.i = n return v def next(self): return self.__next__() class Sentinel: def __init__(self, msg): self.msg = msg def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.msg,) def __init__(self, initialdata={}, maxsize=128): precondition(maxsize > 0) self.m = maxsize+2 # The +2 is for the head and tail nodes. self.d = {} # k: k, v: [v, prev, next,] # the dict self.hs = LRUCache.Sentinel("hs") self.ts = LRUCache.Sentinel("ts") self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. self.update(initialdata) assert self._assert_invariants() def __repr_n__(self, n=None): s = ["{",] try: iter = self.iteritems() x = iter.next() s.append(str(x[0])); s.append(": "); s.append(str(x[1])) i = 1 while (n is None) or (i < n): x = iter.next() s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1])) except StopIteration: pass s.append("}") return ''.join(s) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) def _assert_invariants(self): _assert(len(self.d) <= self.m, "Size is required to be <= maxsize.", len(self.d), self.m) _assert((len(self.d) > 2) == (self.d[self.hs][2] is not self.ts) == (self.d[self.ts][1] is not self.hs), "Head and tail point to something other than each other if and only if there is at least one element in the dictionary.", self.hs, self.ts, len(self.d)) foundprevsentinel = 0 foundnextsentinel = 0 for (k, (v, p, n,)) in self.d.items(): _assert(v not in (self.hs, self.ts,)) _assert(p is not self.ts, "A reference to the tail sentinel may not appear in prev.", k, v, p, n) _assert(n is not self.hs, "A reference to the head sentinel may not appear in next.", k, v, p, n) _assert(p in self.d, "Each prev is required to appear as a key in the dict.", k, v, p, n) _assert(n in self.d, "Each next is required to appear as a key in the dict.", k, v, p, n) if p is self.hs: foundprevsentinel += 1 _assert(foundprevsentinel <= 2, "No more than two references to the head sentinel may appear as a prev.", k, v, p, n) if n is self.ts: foundnextsentinel += 1 _assert(foundnextsentinel <= 2, "No more than one reference to the tail sentinel may appear as a next.", k, v, p, n) _assert(foundprevsentinel == 2, "A reference to the head sentinel is required appear as a prev (plus a self-referential reference).") _assert(foundnextsentinel == 2, "A reference to the tail sentinel is required appear as a next (plus a self-referential reference).") count = 0 for (k, v,) in self.iteritems(): _assert(k not in (self.hs, self.ts,)) count += 1 _assert(count == len(self.d)-2, count, len(self.d)) # -2 for the sentinels return True def freshen(self, k, strictkey=False): assert self._assert_invariants() if k not in self.d: if strictkey: raise KeyError(k) return node = self.d[k] # relink self.d[node[1]][2] = node[2] self.d[node[2]][1] = node[1] # move to front hnode = self.d[self.hs] node[1] = self.hs node[2] = hnode[2] hnode[2] = k self.d[node[2]][1] = k assert self._assert_invariants() def iteritems(self): return LRUCache.ItemIterator(self) def itervalues(self): return LRUCache.ValIterator(self) def iterkeys(self): return self.__iter__() def __iter__(self): return LRUCache.KeyIterator(self) def __getitem__(self, key, default=None, strictkey=True): node = self.d.get(key) if not node: if strictkey: raise KeyError(key) return default self.freshen(key) return node[0] def __setitem__(self, k, v=None): assert self._assert_invariants() node = self.d.get(k) if node: node[0] = v self.freshen(k) return if len(self.d) == self.m: # If this insert is going to increase the size of the cache to # bigger than maxsize. self.pop() hnode = self.d[self.hs] n = hnode[2] self.d[k] = [v, self.hs, n,] hnode[2] = k self.d[n][1] = k assert self._assert_invariants() return v def __delitem__(self, key, default=None, strictkey=True): """ @param strictkey: True if you want a KeyError in the case that key is not there, False if you want a reference to default in the case that key is not there @param default: the object to return if key is not there; This is ignored if strictkey. @return: the value removed or default if there is not item by that key and strictkey is False """ assert self._assert_invariants() if key in self.d: node = self.d[key] # relink self.d[node[1]][2] = node[2] self.d[node[2]][1] = node[1] del self.d[key] assert self._assert_invariants() return node[0] elif strictkey: assert self._assert_invariants() raise KeyError(key) else: assert self._assert_invariants() return default def has_key(self, key): assert self._assert_invariants() if key in self.d: self.freshen(key) assert self._assert_invariants() return True else: assert self._assert_invariants() return False def clear(self): assert self._assert_invariants() self.d.clear() self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. assert self._assert_invariants() def update(self, otherdict): """ @return: self """ assert self._assert_invariants() if len(otherdict) >= (self.m-2): # -2 for the sentinel nodes # optimization self.clear() assert self._assert_invariants() i = iter(otherdict.items()) try: while len(self.d) < self.m: (k, v,) = next(i) assert self._assert_invariants() self[k] = v assert self._assert_invariants() return self except StopIteration: _assert(False, "Internal error -- this should never have happened since the while loop should have terminated first.") return self for (k, v,) in otherdict.items(): assert self._assert_invariants() self[k] = v assert self._assert_invariants() def pop(self): assert self._assert_invariants() if len(self.d) < 2: # the +2 is for the sentinels raise KeyError('popitem(): dictionary is empty') k = self.d[self.ts][1] self.remove(k) assert self._assert_invariants() return k def popitem(self): assert self._assert_invariants() if len(self.d) < 2: # the +2 is for the sentinels raise KeyError('popitem(): dictionary is empty') k = self.d[self.ts][1] val = self.remove(k) assert self._assert_invariants() return (k, val,) def keys_unsorted(self): assert self._assert_invariants() t = self.d.copy() del t[self.hs] del t[self.ts] assert self._assert_invariants() return t.keys() def keys(self): res = [None] * len(self) i = 0 for k in self.iterkeys(): res[i] = k i += 1 return res def values_unsorted(self): assert self._assert_invariants() t = self.d.copy() del t[self.hs] del t[self.ts] assert self._assert_invariants() return map(operator.__getitem__, t.values(), [0]*len(t)) def values(self): res = [None] * len(self) i = 0 for v in self.itervalues(): res[i] = v i += 1 return res def items(self): res = [None] * len(self) i = 0 for it in self.iteritems(): res[i] = it i += 1 return res def __len__(self): return len(self.d) - 2 def insert(self, key, val=None): assert self._assert_invariants() result = self.__setitem__(key, val) assert self._assert_invariants() return result def setdefault(self, key, default=None): assert self._assert_invariants() if not self.has_key(key): self[key] = default assert self._assert_invariants() return self[key] def get(self, key, default=None): return self.__getitem__(key, default, strictkey=False) def remove(self, key, default=None, strictkey=True): assert self._assert_invariants() result = self.__delitem__(key, default, strictkey) assert self._assert_invariants() return result class SmallLRUCache(dict): """ SmallLRUCache is faster than LRUCache for small sets. How small? That depends on your machine and which operations you use most often. Use performance profiling to determine whether the cache class that you are using makes any difference to the performance of your program, and if it does, then run "quick_bench()" in test/test_cache.py to see which cache implementation is faster for the size of your datasets. A simple least-recently-used cache. It keeps an LRU queue, and when the number of items in the cache reaches maxsize, it removes the least recently used item. "Looking" at an item or a key such as with "has_key()" makes that item become the most recently used item. You can also use "refresh()" to explicitly make an item become the most recently used item. Adding an item that is already in the dict *does* make it the most- recently-used item although it does not change the state of the dict itself. """ class ItemIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def next(self): precondition(self.i <= len(self.c._lru), "The iterated SmallLRUCache doesn't have this many elements. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c) if self.i == len(self.c._lru): raise StopIteration() precondition(dict.__contains__(self.c, self.c._lru[self.i]), "The iterated SmallLRUCache doesn't have this key. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) k = self.c._lru[self.i] self.i += 1 return (k, dict.__getitem__(self.c, k),) class KeyIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def next(self): precondition(self.i <= len(self.c._lru), "The iterated SmallLRUCache doesn't have this many elements. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c) if self.i == len(self.c._lru): raise StopIteration() precondition(dict.__contains__(self.c, self.c._lru[self.i]), "The iterated SmallLRUCache doesn't have this key. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) k = self.c._lru[self.i] self.i += 1 return k class ValueIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def next(self): precondition(self.i <= len(self.c._lru), "The iterated SmallLRUCache doesn't have this many elements. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c) if self.i == len(self.c._lru): raise StopIteration() precondition(dict.__contains__(self.c, self.c._lru[self.i]), "The iterated SmallLRUCache doesn't have this key. Most likely this is because someone altered the contents of the LRUCache while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) k = self.c._lru[self.i] self.i += 1 return dict.__getitem__(self.c, k) def __init__(self, initialdata={}, maxsize=128): dict.__init__(self, initialdata) self._lru = list(initialdata.keys()) # contains keys self._maxsize = maxsize over = len(self) - self._maxsize if over > 0: for k in self._lru[:over]: dict.__delitem__(self, k) del self._lru[:over] assert self._assert_invariants() def _assert_invariants(self): _assert(len(self._lru) <= self._maxsize, "Size is required to be <= maxsize.") _assert(len([x for x in self._lru if dict.__contains__(self, x)]) == len(self._lru), "Each key in self._lru is required to be in dict.", filter(lambda x: not dict.__contains__(self, x), self._lru), len(self._lru), self._lru, len(self), self) _assert(len([x for x in self.keys() if x in self._lru]) == len(self), "Each key in dict is required to be in self._lru.", [x for x in self.keys() if x not in self._lru], len(self._lru), self._lru, len(self), self) _assert(len(self._lru) == len(self), "internal consistency", filter(lambda x: x not in self.keys(), self._lru), len(self._lru), self._lru, len(self), self) _assert(len(self._lru) <= self._maxsize, "internal consistency", len(self._lru), self._lru, self._maxsize) return True def insert(self, key, item=None): assert self._assert_invariants() result = self.__setitem__(key, item) assert self._assert_invariants() return result def setdefault(self, key, default=None): assert self._assert_invariants() if not self.has_key(key): self[key] = default assert self._assert_invariants() return self[key] def __setitem__(self, key, item=None): assert self._assert_invariants() if dict.__contains__(self, key): self._lru.remove(key) else: if len(self._lru) == self._maxsize: # If this insert is going to increase the size of the cache to bigger than maxsize: killkey = self._lru.pop(0) dict.__delitem__(self, killkey) dict.__setitem__(self, key, item) self._lru.append(key) assert self._assert_invariants() return item def remove(self, key, default=None, strictkey=True): assert self._assert_invariants() result = self.__delitem__(key, default, strictkey) assert self._assert_invariants() return result def __delitem__(self, key, default=None, strictkey=True): """ @param strictkey: True if you want a KeyError in the case that key is not there, False if you want a reference to default in the case that key is not there @param default: the object to return if key is not there; This is ignored if strictkey. @return: the object removed or default if there is not item by that key and strictkey is False """ assert self._assert_invariants() if dict.__contains__(self, key): val = dict.__getitem__(self, key) dict.__delitem__(self, key) self._lru.remove(key) assert self._assert_invariants() return val elif strictkey: assert self._assert_invariants() raise KeyError(key) else: assert self._assert_invariants() return default def clear(self): assert self._assert_invariants() dict.clear(self) self._lru = [] assert self._assert_invariants() def update(self, otherdict): """ @return: self """ assert self._assert_invariants() if len(otherdict) > self._maxsize: # Handling this special case here makes it possible to implement the # other more common cases faster below. dict.clear(self) self._lru = [] if self._maxsize > (len(otherdict) - self._maxsize): dict.update(self, otherdict) while len(self) > self._maxsize: dict.popitem(self) else: for k, v, in otherdict.items(): if len(self) == self._maxsize: break dict.__setitem__(self, k, v) self._lru = dict.keys(self) assert self._assert_invariants() return self for k in otherdict: if dict.__contains__(self, k): self._lru.remove(k) self._lru.extend(otherdict.keys()) dict.update(self, otherdict) over = len(self) - self._maxsize if over > 0: map(dict.__delitem__, [self]*over, self._lru[:over]) del self._lru[:over] assert self._assert_invariants() return self def has_key(self, key): assert self._assert_invariants() if dict.__contains__(self, key): assert key in self._lru, "key: %s, self._lru: %s" % tuple(map(hr, (key, self._lru,))) self._lru.remove(key) self._lru.append(key) assert self._assert_invariants() return True else: assert self._assert_invariants() return False def refresh(self, key, strictkey=True): """ @param strictkey: raise a KeyError exception if key isn't present """ assert self._assert_invariants() if not dict.__contains__(self, key): if strictkey: raise KeyError(key) return self._lru.remove(key) self._lru.append(key) def popitem(self): if not self._lru: raise KeyError('popitem(): dictionary is empty') k = self._lru[-1] obj = self.remove(k) return (k, obj,) def iteritems(self): return SmallLRUCache.ItemIterator(self) def iterkeys(self): return SmallLRUCache.KeyIterator(self) def itervalues(self): return SmallLRUCache.ValueIterator(self) class LinkedListLRUCache: """ This is slower and less featureful than LRUCache. It is included here for comparison purposes. Implementation of a length-limited O(1) LRU queue. Built for and used by PyPE: http://pype.sourceforge.net useful methods and _assert_invariant added by Zooko for testing and benchmarking purposes """ class Node: def __init__(self, prev, me): self.prev = prev self.me = me self.next = None def __init__(self, initialdata={}, maxsize=128): self._maxsize = max(maxsize, 1) self.d = {} self.first = None self.last = None for key, value in initialdata.items(): self[key] = value def clear(self): self.d = {} self.first = None self.last = None def update(self, otherdict): for (k, v,) in otherdict.items(): self[k] = v def setdefault(self, key, default=None): if not self.has_key(key): self[key] = default return self[key] def _assert_invariants(self): def lliterkeys(self): cur = self.first while cur != None: cur2 = cur.next yield cur.me[0] cur = cur2 def lllen(self): # Ugh. acc = 0 for x in lliterkeys(self): acc += 1 return acc def llhaskey(self, key): # Ugh. for x in lliterkeys(self): if x is key: return True return False for k in lliterkeys(self): _assert(self.d.has_key(k), "Each key in the linked list is required to be in the dict.", k) for k in self.d.iterkeys(): _assert(llhaskey(self, k), "Each key in the dict is required to be in the linked list.", k) _assert(lllen(self) == len(self.d), "internal consistency", self, self.d) _assert(len(self.d) <= self._maxsize, "Size is required to be <= maxsize.") return True def __contains__(self, obj): return obj in self.d def has_key(self, key): return self.__contains__(key) def __getitem__(self, obj): a = self.d[obj].me self[a[0]] = a[1] return a[1] def get(self, key, default=None, strictkey=False): if not self.has_key(key) and strictkey: raise KeyError(key) if self.has_key(key): return self.__getitem__(key) else: return default def __setitem__(self, obj, val): if obj in self.d: del self[obj] nobj = self.Node(self.last, (obj, val)) if self.first is None: self.first = nobj if self.last: self.last.next = nobj self.last = nobj self.d[obj] = nobj if len(self.d) > self._maxsize: if self.first == self.last: self.first = None self.last = None return a = self.first a.next.prev = None self.first = a.next a.next = None del self.d[a.me[0]] del a def insert(self, key, item=None): return self.__setitem__(key, item) def __delitem__(self, obj, default=None, strictkey=True): if self.d.has_key(obj): nobj = self.d[obj] if nobj.prev: nobj.prev.next = nobj.next else: self.first = nobj.next if nobj.next: nobj.next.prev = nobj.prev else: self.last = nobj.prev val = self.d[obj] del self.d[obj] return val.me[1] elif strictkey: raise KeyError(obj) else: return default def remove(self, obj, default=None, strictkey=True): return self.__delitem__(obj, default=default, strictkey=strictkey) def __iter__(self): cur = self.first while cur != None: cur2 = cur.next yield cur.me[1] cur = cur2 def iteritems(self): cur = self.first while cur != None: cur2 = cur.next yield cur.me cur = cur2 def iterkeys(self): return iter(self.d) def itervalues(self): for i,j in self.iteritems(): yield j def values(self): l = [] for v in self.itervalues(): l.append(v) return l def keys(self): return self.d.keys() def __len__(self): return self.d.__len__() def popitem(self): i = self.last.me obj = self.remove(i[0]) return obj pyutil-3.3.2/pyutil/data/000077500000000000000000000000001437014040400153015ustar00rootroot00000000000000pyutil-3.3.2/pyutil/data/wordlist.txt000066400000000000000000001437571437014040400177320ustar00rootroot00000000000000abacus abandon abbey abbot abdomen abduct abet abhor abolish abort abound abrupt abscess abscond absent absorb abstain abstract absurd abyss accent accept access accident acclaim accompanist accomplish accord accordion accost account accredit accustom ace acid acorn acquaint acquit acrid acrobat acronym act action activist actor actress actual acumen ad adapt add addict address adept adjoin adjourn adjunct adjust admit admonish ado adopt adorn adroit adult adulthood adverb advert advisor aeon aerial aerosol affair affect affection affidavit affirm affix afflict afford affront african aftereffect aftermath afternoon afterthought afterward age agenda agent aggressor agnostic agreement aid aids ail ailment aim aimless air aircraft airfield airmail airport airstrip alarm alarmist albino album alcohol ale alert alga algebra algorithm alias alibi alien alight align alkali all allah allay alley allot allow alloy almanac almond alphabet altar alter alto altruism aluminum amass amateur ambassador amber ambush ameba ameer amen amend american amethyst amir ammonia amnesia amoeba amount amp ampersand amphibian amulet anaemia anaesthesia analog analyst anarchist ancestor anchor ancient android anemia anesthesia angel anger angler anguish ani annex annoy annual annul anoint anorak answer ant antagonist antenna anthem anthropologist antic anticlimax antler antonym anus anvil anxious aorta apart apartheid ape apex aplomb appal apparatus apparel appeal appear append appendix applaud appoint apprehend apprenticeship approach apricot april apron apt aquarium aqueduct arbor arc arch archaeologist archbishop archeologist archer archipelago architect archway ardent ardor arduous are area arena argument aria aristocrat ark arm armadillo armament armchair armor armpit aroma arraign array arrest arrow arsenal arsenic arson art artefact artifact artisan artist ascend ascent ascertain ash ashtray asian ask asparagus aspect aspen asphalt aspirin ass assail assassin assault assent assert assess assessor asset assign assist assort asterisk asteroid asthma astonish astound astronaut asylum atheism atheist atlas atom attach attack attain attempt attend attest attic attorney attract auburn auction audio audit auditor auditorium augment august aunt aura auto autocrat autograph autumn avail avert avocado avoid avow await awaken award awe awful awkward ax axe axiom ay aye azalea babe baboon bachelor back backer backgammon background backhand backlash backlog backpack backtrack backward bacon bacteria bacterium bad badger badminton bag bagel bail bait bake baker bald bale balk ball ballad ballast ballerina ballet balloon ballot ballroom balm baloney bamboo ban banana band bandana bandanna bandit bandstand bandwagon bang banish banjo bank banker bankrupt banner banquet banter baptism bar barb barbarian barber bard bare bargain bark barley barn barnyard baron barrel barren barrier barter base basement bash basic basil basin bask basket bass bassoon bastard bat batch bath bathroom bathtub baton battalion batter battlefield battleship baud bawl bay bayonet bayou bazaar be beach beacon bead beak beaker beam bean bear beard bearer beast beat beater beautician beaver beckon bed bedbug bedlam bedrock bedroom bedspread bee beech beef beer beeswax beet befit befriend beg beggar begin behalf behavior behead behind behold belabor belch belief bell bellboy bellhop bellow belong belt bemoan bench bend benefactor benefit bent bequeath bequest beret berth beseech beset best bestial bestow bet betray better bettor bewilder bewitch bias bib bicker bid bide biennial bigamist bigot bike bikini bile bill billboard billfold billion billow bin bind binder bingo biologist birch bird birth birthday birthmark biscuit bisect bishop bison bit bitch bite bitter bittersweet blab black blackbird blackboard blacken blackhead blackjack blacklist blackmail blackout blacksmith blacktop bladder blade blame blanch blank blanket blare blast blatant blaze blazer bleach bleat bleed blemish blend bless blight blimp blind blindfold blink blinker blip bliss blister blitz blizzard blob bloc block blockhead blog blogger blond blood bloodhound bloom blossom blot blotch blotter blow blowout blowtorch blubber bludgeon blue bluebird bluegrass blueprint bluff blunder blunt blur blurt blush bluster boa boar board boarder boardwalk boast boat bob bobbin bobcat bobsled bode bodyguard bog boil boiler bold bologna boloney bolster bolt bomb bombard bomber bond bone boney bonnet bonus boo book bookend booklet bookmark bookshop bookworm boom boomerang boon boor boost booster boot booth bootleg border bore boredom born borough borrow bosom boss botanist botch bother bottleneck bottom bough boulder boulevard bound bouquet bourbon bout bow bowel bowl bowlder box boxcar boxer boy boycott boyfriend boyhood bra brace bracelet bracket brag braggart braid brain brainstorm brainwash brake bran branch brand brandish brass brassier brat bravado brave bravo brawl brawn bray brazen brazier breach bread breadth break breakdown breakfast breakthrough breast breath breather breed breeder brew bribe brick bridal bride bridegroom bridesmaid brief bright brighten brilliant brim brine bring brink brisk british broach broad broadcast broaden broccoli broil broiler broker broncho bronco brooch brood brook broom broth brother brotherhood brow browbeat brown brows brunch brunt brush brutal brute buck bucket bud budget buff buffalo buffer buffet buffoon bug bugger bugler build builder bulb bulk bull bulldog bullet bulletin bullfight bullfrog bullion bum bump bumper bun bunch bungalow bungler bunion bunk bunker burden bureau bureaucrat burger burglar burial burlap burn burner burnish burp burr burro burrow burst bus bush bushel businessman businesswoman buss bust but butcher butler butt butter buttercup buttermilk butterscotch buttock button buttress buy buyer buzz buzzard buzzer by bye bypass byte byway cab cabaret cabin cabinet cacao cactus cadet cafeteria cage cake calcium calculus calendar calf calico calk call caller callous callus calm camel cameo camera camp campaign camper campus can canal cancel cancer candid candlestick candor cane canker cannon canon cantaloup canteen canter canvas canvass canyon cap capacitor cape caper capitalist captain caption captor car caramel carat caravan carbon carburetor carcass card cardboard cardigan care career careful careless caress cargo caribou carol carp carpet carrier carrion carrot cart cartel carton cartoon cartoonist cartwheel case cash cashew cashier casino cask casket cassino cast castaway caster castoff casual cat cataclysm catalog catapult cataract catch catchup cater caterpillar catfish catholic catnap catnip catsup catwalk caucus caulk causal causeway caustic caution cautious cave caveat cavern caviar cavort caw ceaseless cedar cede cell cellar cellist cello cellular cement censor censorship census cent center central cereal certain chafe chaff chagrin chain chair chairman chairperson chalet chalk chamber chameleon champ champion championship chancellor channel chant chap chapel chaperon chaplain chapter char character charcoal chariot charisma charlatan charm chart charter chase chasm chasten chat chatter chatterbox chauffeur chauvinist cheap cheapen cheat check checker checkup cheek cheep cheer cheerful cheesecloth cheetah chef chemist cherish cherub cherubim chess chest chestnut chew chic chick chicken chide chief chieftain child childbirth childhood chile chili chill chilli chime chimney chimp chin china chink chintz chip chipmunk chipper chiropractor chirp chisel chloroform choir choke cholera chop chopper choral chord chore chorus chow chowder christen christian christmas chrome chromium chronic chrysanthemum chuck chug chum chunk church churn chute cider cigar cigaret cinch cinder cinema cinnamon cipher circuit circular circumvent circus cistern cite citizen citizenship citrus civic civil civilian clack claim clam clamber clamor clamp clan clang clank clap clapper clarinet clash clasp class classic classroom clatter claustrophobia claw clay clean cleaner cleans cleanser clear cleat cleaver clef cleft clench clergyman cleric clerk clever click client cliff climax climb climber clime clinch cling clinic clink clip clipboard cloak clock clockwork clod clog cloister clone close closet clot cloth clothespin cloud cloudburst clout clove clover clown club cluck clue clump cluster clutch clutter coach coal coarsen coast coaster coat coax cob cobalt cobbler cobra cobweb cock cockpit cockroach cocktail cocoa cocoanut coconut cocoon cod code coercion coexist coffer coffin cog cognac coil coin coke cold colic collar collect collector colon colonel color colt column coma comb combat come comedian comet comfort comic comma command commando commend comment commission commit common commonwealth commune communion communism communist compact companion companionship comparison compass compassion compatriot compel competitor complain complaint complement complex complexion compliment compost compound comprehend compress computer comradeship con conceal conceit concept concern concert concerto concoct concord concur condemn condescend condiment condom condominium condor conduct conductor cone confer confess confetti confirm conflict conform confound confront congeal congest congress congressman congresswoman conifer connect connector connoisseur conquer conqueror conquest conscious consensus consent consider consign consist consort constant constrain constraint constrict construct consul consult consumer contact contagion contain contempt contend content contest context continent contort contour contraband contract contractor contradict contrast contributor control convent convert convertor convey convict convoy coo cook cookbook cool cooler coop cooper cop cope copier copious copper copperhead coral cord cordial cordon corduroy core cork corkscrew corn cornea corner cornet cornmeal cornstarch corps corral correct correspond corridor corrupt corset cosmonaut cosmopolitan cosmos cost cot cotton cottontail cottonwood couch cougar cough council councillor councilor counsel counsellor counselor count countdown counter counteract counterattack counterfeit counterpart countersign countess countryman coup coupon courier court courteous courtroom courtship courtyard cousin cove cover covert covet cow coward cowboy cower cowgirl crab crack cracker crackpot craft craftsman crag cram cramp crane cranium crank crash crate crater crave crawfish crawl crayfish crayon craze creak cream creation creator credit creditor creed creek creep crepe crescendo crescent crest cretin crew crib cricket crime crimson crisp crisscross criterion critic croak crochet crock crocus crook crooked croon crop croquet cross crossbow crossroad crosswalk crossword crotch crouch crow crowbar crowd crown crucial crucifix crucifixion crude cruiser crumb crunch crush crust crustacean crutch crux crypt crystal cub cube cuckoo cue cuff cull culprit cult cunning cup cupboard cups cur curb curd cure curfew curio curios curious curl currant current curriculum curtail curtain curtsey cushion custard custodian custom cut cutback cutlet cutter cutthroat cyclist cymbal cynic cypher cypress cyst czar dab dachshund dad daffodil dagger dais dam dame damn damp dampen damsel dancer dandelion dandruff danger dapper dare daredevil dark darken darn dart dash dashboard date datum daub daughter daunt dawn day daybreak daydream daylight daze deacon dead deaden deadlock deaf deal dealer dean dear dearth death debit debrief debt debtor debug debunk debut decay deceit decent decibel deck decorum decoy deduct deed deem deep deepen deer default defeat defect defend defer defiant deficit deflect deform defraud defrost deft deign deject delay deli delicatessen delight delimit delirium deliver delta demagog demand demean demeanor demerit democrat demolish demon den denial denim dens dent dentist depart department depend depict deport deposit depot depress depth derail derelict derrick descend descent descriptor desert design desist desk despair despatch despot dessert destroy detach detail detain detect detector deter detest detour detract detriment develop devil devote devour devout dew diagram dial dialect dialog diamond diaper diaphragm dice dictatorship diction die diesel diet differ dig digest digit digress dike dilemma dill dim dime dimension diminish din dine diner dinner dinosaur dip diphtheria diphthong diploma diplomat direct director dirt disallow disappear disappoint disarm disarray disavow disband disbelief disc discard discern disciplinarian disclaim disco discomfort disconcert disconnect discord discount discredit discreet discus discuss disdain disembark disfavor disgust dish dishearten dishonest dishonor disillusion disinfect disinherit disjoint disk dismal dismay dismiss dismount disobey disown dispatch dispel display dispossess disquiet disregard disrepair disrespect disrupt dissect dissent dissimilar distant distend distil distinct distinguish distort distract distress distributor district distrust disturb ditch dither ditto dive diver divers divert divest dividend divisor do dock doctor document doe doer dog dogma dogwood dole doll dollar dolphin domain dome dominion domino don donkey donor donut doom door doorman doorstep doorway dope dose dot dote doubt dough doughnut dove down downpour downtown downward doze dozen drab draft draftsman drag dragon drain drama dramatist drape drastic draw drawback drawer drawl dread dream dreamer drench dress dresser drier drift driftwood drill drink drinker drip drive drivel driver driveway drone drool droop drop dropout drought drouth drove drown drug druggist drum drummer drumstick drunk drunkard drunken dryer dryness dub dubious duchess duck duct dud dude due duel duet dugout duke dull dumbfound dumfound dump dune dungeon dunk dupe duplex duress dusk dust dustpan dutch dwarf dwell dweller dye dynamo eager ear eardrum earl earmark earn earner earnest earring earshot earth earthworm easel east easter eat eavesdrop ebb echo ecologist economist ecosystem eczema edict edit editor eel effect effort effortless egg eggplant ego egoism egotist eight eighteen eighteenth eighth eightieth eject eke elbow elder elect elector electrician electron element eleven eleventh elf elicit elk elm email embalm embargo embark embarrass embed embellish ember emblem emboss embryo emerald emir emit emperor emphysema employ employe emporium empress enact enamel enchant encompass encroach encyclopaedia encyclopedia end endear endeavor endless endow enema english engross engulf enigma enjoy enlighten enlist enliven enough enrich enrol ensign entail enter entertain enthral enthusiasm enthusiast entomologist entrant entrap entreat entrench entrust envelop envious environment envoy eon epaulet epic epidermis epilog epitaph epithet epoch equal equestrian equilibrium equinox equip era eras erect err errand error erupt escort eskimo esophagus essay establish esteem etch ether ethic ethnic european evangelist eve even event evergreen evict evil ewe exact exalt exam exceed excel except excerpt excess exclaim execution executor exempt exert exhaust exhibit exhort exist exit exodus expand expect expel expend experiment expert explain explicit exploit export expound express expressway extend extent exterior extinct extinguish extol extort extortion extra extract extravert extremist extrovert exult eye eyebrow eyelash eyelid eyesight fabric face facet facial fact faction factor factual fad fade faggot fagot fail faint fair faith fake falcon fall fallout falsehood falsetto falter fame familiar fan fang fare farm farmer fascism fascist fashion fast fasten fat fatal fate father fatherhood fatherland fathom fatten faucet fault fauna favor fawn faze fear fearless feast feat feather fed federalist fee feed feedback feeder feel feeler feign feint fell fellow fellowship felon felt feminist fen fend fender ferment fern ferret fervent fervor fester festoon fetch fetich fetish fetter fetus feud feudal fever feverish few fez fiasco fib fibber fiber fiberglass fiction fiddler fidget field fiend fiendish fiesta fifteen fifteenth fifth fiftieth fig fight fighter figment figurehead filament filch file filet fill fillet film filter filth fin final finalist finch find fine finger fingernail fingerprint fingertip finish fir fire firearm fireman fireproof firewood firework firm first fiscal fish fisherman fission fist fit five fix fizz flag flagrant flagship flail flair flake flame flamingo flank flannel flap flapjack flare flash flashback flashlight flask flat flatten flatter flaunt flavor flaw flawless flea fleck flee fleet flesh flex flick flicker flier flight flinch fling flint flip flipper flirt flit float flock flog flood floodlight floor flop flora florist floss flotilla flounder flour flourish flout flow flower flu flue fluff fluid fluke flunk flunkey flush fluster flute flutist flutter flux flyer flyover foal foam focus fodder foe foetus fog fogey foghorn foil foist fold folder folk follow foment fond font food foodstuff fool foolish foot foothold footpath footprint footstep footstool footwear footwork foray forbad forbear forbid ford fore forearm forecast forefront forego foreground forehead foreign foreleg foreman foreshadow foresight forest foreswear forethought forewarn foreword forfeit forger forget forgo fork form formal format former formula forswear fort fortieth fortnight fortress forum forward fossil foster foul found founder fount fountain four fourteen fourteenth fourth fowl fox foyer fracas fraction fragment frame framework franc frank frantic fraud fray freak free freedom freeway freezer freight freighter frequent fresh freshen freshman fret friar friction friday friend friendship fright frighten frigid frill frisk fritter frock frog frolic frond front frontier frost frostbit froth frown frugal fruit fruition fruitless fuel fulcrum fulfil full fume fun function fund fundamentalist fungus funnel fur furious furl furlong furlough furnish furor furrow further fuse fusion fuss fuze fuzz gab gadget gag gage gain gait gal gala gale gall gallant galley gallon gallop gallows gambit gambler game gamut gander gang gangplank gangster gangway gap gape garb garden gardenia garland garlic garment garnet garnish garret garrison garter gas gash gasket gasp gate gateway gather gauntlet gavel gawk gay gaze gear gee gees gelatin geld gem gender gene general generic generous genes geneticist genial genius gent gentleman genus geologist geranium gerbil germ get getaway geyser ghetto ghost ghoul giant gibber gibberish gibe gift gig gild gill gilt gimmick gin ginger gingerbread gingham girder girl girlfriend girlhood girth gist give given gizzard glacier glad gladden glade glamor glamour gland glare glass glaze gleam glean glee glen glide glider glimmer glint glisten glitter gloat global globe gloom glorious gloss glove glow glower glue glut glutton glycerin gnarl gnash gnat gnaw gnome gnu go goad goal goat gob goblet goblin god godchild goddess godsend gold golden goldfish goldsmith golf golfer gondola goner gong goo good goodby goof goon gopher gore gorilla gospel gossip goulash gourd gourmet gout govern government governor gown grab grace gracious grade gradient gradual graffito graft grain gram grammar grand grandchild grandeur grandson grandstand granola grant grape grapefruit graph graphic grasp grass grate grater grave gravel graveyard gray graze great greed greek green greenback greenhorn greet gremlin grey greyhound grid gridiron grief grill grim grime grin grind grinder grip gripe grit groan grocer groin groom grope gross grotto grouch ground groundwork group grouper grove grovel grow grower growl growth grub gruel grunt guarantor guard guardian guerilla guerrilla guess guest guffaw guidebook guild guilt guitar gulch gulf gull gullet gulley gulp gum gumdrop gumption gun gunman gunner gunshot guru gush gusher gust gut gutter guy gybe gym gymnasium gymnast gynecologist habit habitat hack hacker hackney hacksaw haddock haemoglobin haemophilia hag hail hair haircut hairdo hale half halibut hall hallelujah hallmark halloween hallway halo halt halter ham hamlet hammer hammock hamper hamster hamstring hand handbag handbook handcuff handed handicap handicraft handiwork handkerchief handlebar handler handout handrail hands hang hangar hanger hangout hanker happen harass harbor hard harden hardship hardwood hare harem hark harlot harm harmless harmonica harp harpist harpoon harpsichord harrow harsh hart harvest hash hasten hat hatch hatchet hate haul haunt have haven havoc hawk hay haystack hazard haze hazel he head header headland headlight headrest headroom headway heal healer health heap hear hears hearsay heart heartbeat heartbreak heartburn hearten hearth heat heater heathen heather heaven heavyweight hebrew heckler hedgehog heed heel heifer height heighten heir heirloom heliport helium hell hello helm helmet help helper helpless hem hemlock hemoglobin hemophilia hemp hen henchman her herald herb herd here hermit hernia hero heroin heroism heron herring hew hexagon heyday hiatus hiccup hick hide hideaway hideous high highbrow highjack highland highlight highway hijack hike hiker hill hilt him hind hinder hindsight hint hinterland hip hippopotamus hire hiss historian hit hitch hive ho hoard hoarder hoax hobgoblin hobnob hobo hock hockey hoe hog hoist hold holder holdup hole holiday holler hollow holocaust holster home homeland homeless homesick homespun homestead homework homey homonym hone honest honey honeycomb honeymoon honk honor hood hoodlum hoodwink hoof hook hoop hoorah hooray hoot hop hope hopeless hopscotch horizon horn hornet horror horseback horseplay horseradish hose host hostel hostess hot hotel hothead hound hour hourglass houseboat household housework hovel hover how howl hub hubbub hue huff hug huge hulk hull hullabaloo hum human humanitarian humdrum humid hummingbird humor humorist hump hunch hunchback hundredth hunger hunk hunt hunter hurl hurrah hurray hurt husband hush husk hustler hut hutch hyacinth hyaena hybrid hydrant hydrogen hyena hymn hymnal hyphen hypnotist hypochondria hypochondriac hysteria ice iceberg icon id idea ideal idealist idiom idiot idol if igloo iguana ikon ill imbed imp impact impair impart impeach impel impend imperfect imperil impetus implant implement implicit import impostor impound impoverish impress impression imprint imprison impromptu in inbreed incest inch incident incisor incognito incorrect increment incur indent index indian indict indigo indirect individualist indoor induct industrialist inertia infant infect infer inferior inferno infest infidel infield infirm inflict influenza influx inform ingest ingrain inhabit inherit inhibit inhuman initial inject ink inland inlay inlet inn inning innuendo input inquest insect insert insight insignia insist insomnia inspect inspector instal instant instep instil instinct institution instruct instructor instrument insulin insult intellect intend intent inter interact intercept intercom interest interim interior interject interlock intern interplay interpret interrupt intersect interview intravenous intrench introvert intrust invalid invent inventor invert invest investor inward ion iota ire iris irk iron irregular islam island isthmus it italic itch item jab jabber jack jackal jackass jacket jackpot jade jagged jaguar jail jailer jailor jam jamb janitor jar jargon jaunt javelin jaw jay jaywalk jazz jealous jeer jell jellyfish jerk jersey jest jester jet jettison jew jewel jibe jig jigsaw jilt jinx job jockey jog jogger join joint joke joker jolt jot journal journalist journey jovial joy joyful joyous judaism judgement judgment judo jug juggler jugular jumbo jump jumper junction june junior junk junket junta juror just jut jute kangaroo karat kayak keel keen keep keeper keg kelp kennel kerchief kernel ketchup key keyboard keyword khaki kick kickback kickoff kid kidnap kidney kill killer kiln kilo kilogram kilowatt kilt kimono kind kindergarten kinfolk king kingdom kink kinship kiosk kiss kit kitchen kite kitten kiwi knack knapsack knead knee kneecap kneel knife knight knighthood knit knob knock knocker knockout knoll knot know koala koran kosher kowtow lab label labor labyrinth lace lack lacquer lad ladder lade ladybug lag laggard lagoon lair lake lamb lame lament lamp lampoon land landlord landmark landslid lane languish languor lantern lap lapel laps lard lark larva larynx laser lash lass last latch late later lateral latex lath lather latin latter laud laugh laughingstock laughter launch launcher launder laurel lava lavish law lawn lawsuit lawyer lay layer layman layout lead leader leadership leaf leaflet leak lean leap leapfrog learn leash least leather lectern ledger leech leek leer leeway left leg legal legend leggin legion lemon lend length lengthen lens lentil leopard leotard leper lesbian lesion less lessen lesson let letdown letter letterhead letup leukemia level lever lexicon liaison liar libel librarian lichen lick lid lie lieu life lifeboat lifeguard lift ligament light lighten lighter lightning lightweight like likelihood liken lilac lilt limb limber lime limelight limerick limit limp linchpin line linear linen liner linger lingo linguist liniment link linoleum lint lion lioness lip lipstick liqueur liquid liquor lisp list listen liter litter litterbug live livelihood liven liver livestock lizard llama load loaf loafer loam loan loath lob lobbyist lobe lobster local lock locker locket locksmith locust lodger loft log logarithm logic loin loincloth loiter loll lollipop lollypop lone long longhand longshoreman look lookout loom loon looney loop loosen loot lop lope lord lore lose loser loss lot lotion lotus loud love lover low lowdown lower loyal luck lug lull lumber lumberjack lump lunch luncheon lung lupin lurch lure lurk lush lust luster lute lye lymph lynch lynchpin lyre lyric ma macaroni mace machinist mackerel mad madam madcap madden madman maelstrom magenta maggot magic magician magnesium magnet magnolia maid maiden mail mailbox mailman maim main mainland mainstay maintain major make maker makeshift makeup malaria male malign mall mallard mallet malt maltreat mama mamma mammal mammoth man mandolin mane manger mango manhood mania maniac manicurist manifest manifesto manifold mankind mannequin manner manor mansion mantel manual manuscript map mar marathon march mare margin marigold marihuana marijuana marina mark marker market marksman maroon marrow marsh marshal marshmallow mart martyr martyrdom marvel marxism marxist mascara mascot mash mask masochist mason mass mast master mastermind mat matador match matchbook mate materialist math mathematician matriarch matrix matron matt matter mattress maul mausoleum maverick maxim maximum may mayhem mayor maze meadow meal mean meander meaning meat medal medallion meddler media medium medley meek meet megaton mellow melodrama melon melt member membership memento memo memoir memorandum mend menial mental menthol mention mentor menu meow merchant merciless mere merger meridian merit mermaid merriment mesh mess metal metaphor mete meteor meteorologist meter method metro metropolis mew miaow microfilm microsecond midday middleman midget midnight midriff midst midstream midway mien might migrant mike mild mildew mile militia milk milkman mill miller milligram million millionth millisecond mime mimic mincemeat mind minded mindless mine miner minibus minimum minion mink minnow minor minstrel mint minuet minus mire mirror mirth misbehavior mischief misconduct misdemeanor misdirect miser misfit mishap misinform misinterpret mislay mislead mismatch misprint misread misrepresent miss mission mist mistaken mistress mistrust misunderstand mite mitt mitten mix mixer moan moat mob moccasin mock mockingbird mode model modern modest modicum mohair moisten molar mold mole molest mollusc mollusk molt mom moment momentum momma monarch monday money mongrel monitor monk monkey monogram monolog monorail monsoon monster month monument moo mood moon moonbeam moonlight moor moos moot mop mope moral moralist morass moratorium more morn moron morsel mortal mortar mosaic moslem mosquito moss most motel moth mother motherhood motif motion motley motor motorist motorway motto mound mount mountain mourn mourner mouth move movement mover mow mower mr ms much muck mucus mud muff muffin muffler mug mugger mulch mule mull mum munch mural murder murmur muse museum mush mushroom music musician musk musket muslim muss mussel must mustang mustard muster mutant mute mutt mutter mutton mutual myriad mystic myth nab nag nail naked name nap napalm nape napkin narrow nasal nation nationalist naturalist naught nausea navel nay nazi near nearsighted neat nebula neck neckerchief nectar need needless needlework neglect neglig neigh neighbor neighborhood neon nephew nervous nest net network neurologist neuron neuter neutral neutron new newborn newscast newsprint newsstand newt next nice nick nickel night nightclub nightgown nil nincompoop nine nineteen nineteenth ninetieth ninth nip nit nite nitrogen nitwit no nobleman noblewoman nod node noiseless nomad nonconformist nonpartisan nonprofit nook noon norm normal north northeast northwest nose nostalgia nostril notch note notebook nothing noticeboard notion nougat nought noun nourish novel novelist now nucleus nude nugget null numb number nun nuptial nursemaid nut nutmeg nutrient nutriment nylon nymph oaf oak oar oath oatmeal obelisk obey object objection objector oblivion oblong obsess obstetrician obstruct obtain obvious occasion occur ocean octagon october octopus ocular odd ode odor of off offbeat offend offer offset offshoot often oh ohm oil ointment ok okay okra old omelet omen omit one onion onomatopoeia onrush onset onslaught onus onward opal open opera operand ophthalmologist opinion opium opossum opportunist oppress oppressor opt optic optician optimist optimum option optometrist opus oral orangutan orangutang orbit orchard orchestra orchid ordain ordeal order ore organ organist orient origin ornament ornithologist orphan orthodontist ostrich other otter our oust ouster out outbreak outburst outcast outclass outdo outdoor outfield outfit outgrow outgrowth outing outlast outlaw outlay outlet outlook outpost output outrun outset outskirt outsmart outstrip outward outweigh outwit oval oven over overbear overburden overcast overcoat overcrowd overdo overdraw overeat overflow overgrow overhand overhang overhaul overhead overhear overheat overlap overlay overload overlook overnight overpass overprint overreact overrun overs overshadow overshoot oversight oversleep overstep overt overthrow overturn overweight overwhelm overwork ovum owe owl own owner ownership ox oxygen oyster pa pace pacifist pack packer packet pact pad paddock padlock pagan page pageant pagoda pail pain painful painless paint pair pal pale paleontologist pall pallor palm palomino pamper pamphlet pan panacea pancreas panda pandemonium pander pane panel pang panic panorama pant panther papa papaya paper paperback paperweight paprika papyrus par paradox paraffin paragon paragraph parakeet parallel paranoid paraphernalia parasol parcel parch parchment pardon pare parent parenthood parish park parka parkway parliament parlor parrakeet parrot pars parsec parsley parsnip parson part partial particular partisan partizan partner partnership pass passageway passbook passion passport password past pasta pastel pastor pat patch patchwork pate patent path pathologist pathway patient patio patriarch patriot patrol patron patter pattern paunch pauper pave pavement pavilion paw pawn pay payer payment payoff pea peach peacock peak peal peanut pear pearl peas peasant peat pecan peck peculiar pedal pedant peddler pedestrian pediatrician pediatrist pedlar peek peel peep peer peg pelican pellet pelt pelvis pen penal pencil pendant pendulum penguin penicillin peninsula penis penmanship pennant pension pentagon peon pep pepper peppermint percent perch perfect perfectionist perform peril period perish perk permit perpendicular perplex persecutor persist person personnel pertain perturb pervert pessimist pest pester pet petal peter petrol petroleum petticoat petunia pew pewter phantom pharmacist phase pheasant phenomenon philanthropist phlegm phobia phone phoney phonograph phosphorus photo photograph photon phrase physic physician physicist pi pianist piano piccolo pick pickaback pickax picket pickpocket pickup picnic pie piecework pier pig pigeon piggyback pigment pigpen pigtail pike pile pilfer pilgrim pill pillar pillow pilot pin pinch pincushion pine pinion pink pinpoint pint pioneer pipe piranha pistachio pistol piston pit pitch pitcher pitchfork piteous pivot pizza placard place placenta placid plagiarist plaid plain plaintiff plan plane planet planetarium plank plankton planner plant plantain planter plasma plaster plastic plate plateau platform platinum platoon platter play player playground playpen playwright plaza plea plead pleas pleasant pleat plight plod plop plot plotter plough plow ploy pluck plug plum plumb plumber plume plummet plump plunder plunger plural plus plush plutonium plywood pneumonia poach poacher pocket pocketbook pockmark pod podium poem poet poetic poinsettia point pointer pointless poison poke poker polar pole policeman policewoman polio polish politician polka poll pollen pollster polo polygon polyp pomp poncho pond ponder pontoon pool poop poor pop popcorn poplar popular porcelain porch pore pork port portal portend portent porter portfolio portico portion portrait portray pose possess possessor possum post postcard poster posterior postman postmark postscript pot potassium potato potion potter pouch pound pour pout powder power powwow prank prawn pray prayer preach preacher precinct precursor predecessor predict preempt preen prefab prefer prefix premier premiss premium prepay present press pretend pretext pretzel prevail prevent preview previous prey price prick pride priest priestess priesthood prim prime primer primp princess print printer printout prior prism prison prize pro probe problem proceed process procession processor proclaim prod product profess profession professor proffer profit profound program progress prohibit project projector proletarian proletariat prolog prolong prom prompt prong pronoun proof proofread prop propaganda propel proper prophet proportion proprietor prose prosecutor prospect prospector prospectus prosper protagonist protect protector protein protest protestor protocol proton protract protractor proud prove proven proverb provision proviso prow prowess prowl prowler prude prune psalm pseudonym psych psychiatrist psychic psychoanalyst psychologist psychopath public publish puck pucker pueblo puff puke pull pulley pulp pulpit puma pummel pump pumpernickel pumpkin pun punch punctual pundit punish punk punt punter pup pupil puppet pure purport purr pursuit pus push pusher puss put putt putter pyramid pyre python quack quadrant quadruplet quail qualm quarrel quart quarter quarterback quartet quartz quash quaver quay queen queer quell quench quest question queue quick quicken quicksand quiet quill quilt quintet quintuplet quip quirk quit quitter quiver quiz quorum quota quotient rabbi rabbit raccoon race racetrack racial racism racist rack racket racoon racquet radar radial radio radish radium radius raft rafter rag ragamuffin rage ragged raid raider rail railroad railway rain rainbow raincoat raindrop rainstorm raisin rake ram rambler ramp ramrod ranch rancher rancor random rang ranger rank ransack ransom rant rap rape rapid rapist rapport rare rascal rash rasp rat rate ratio ration rational rattler raucous rave ravel raven ravish raw ray rayon raze razor re reach react reaction reactor read reader readjust real realism realist realm ream reap reaper reappear rear reason rebel rebellion rebind rebirth rebound rebuff rebuild rebut recant recap receipt recent receptionist recess reckless reckon reclaim recoil recollect recommend reconnect reconstruct record recount recoup recover recruit rector rectum recur recurs red redden redeem redesign redhead redirect redo redress reed reef reek reel reelect refer referendum reflect reflector reflex reform reformat refrain refresh refuel refuge refund refurbish regain regal regalia regard regatta regent regimen regiment region registrar regress regret regular rehash reign rein reindeer reject rejoin relationship relax relay relent relentless relic relief religion relinquish relish reload remain remark remind remiss remit remnant remodel rend render rendezvous renew renown rent rental reopen repair repay repeal repeat repel repent replenish replica report represent repress reprimand reprint reproach reprogram republican request reread rescind rescuer research resent reservoir reset resign resin resist resistor resort resound respect respond rest restart restful restless restrain restraint restrict resubmit result resurrect retail retain retard retch retina retort retract retreat retrospect return reunion rev revamp reveal revel revert review revisit revolt revolution reward rhino rhinoceros rhododendron rhubarb rhyme rhythm rib ribbon rice rich ricksha rickshaw ricochet rid ride rider rift rig right righteous rigid rigor rile rim rind ring ringlet ringworm rink riot rioter rip ripe ripen rise riser risk rite ritual rival river rivet roach road roadblock roam roar roast rob robber robe robin robot robust rock rocker rocket rod rodent rodeo roe role roll roller romp roof rook room roost rooster root rope rose roster rostrum rot rote rotor rotten rotunda rough roughen round roundabout rout row rowboat royal rub rubber rubbish ruckus rudder rude rue ruff ruffian rug rugged ruin rule ruler rum rumor rump run runaway rundown rune rung runner runt runway ruse rush rust rustic rustler rut ruthless rye sabbath saber saboteur sac sack sacrament sad sadden sadism sadist safari safe safeguard saffron sag saga sage sagebrush sail sailboat sailor saint sake saki salad salami sale salesman salesperson saleswoman salient saliva sallow salmon salon saloon salt same sanatorium sanction sand sandal sandbag sandman sandstorm sandwich sang sanitarium sap sarcasm sari sash satan satchel satin satirist saturday saucepan saucer sauerkraut sauna saunter save savior saviour savor saw sawdust say scab scaffold scalar scald scale scallop scalp scalpel scamper scan scandal scanner scant scapegoat scar scare scarecrow scarf scarlet scatter scatterbrain scenario scene scent scepter scheme schemer scholar scholarship school schoolboy schoolchild schooner scientist scissor scoff scold scollop scoop scoot scooter scope scorch score scorn scorpion scotch scoundrel scour scout scowl scram scrap scrapbook scrape scratch scrawl scream screech screen screw scribe script scroll scrub scruff scuff sculptor scum sea seafood seal seam seaman seamstress seaport sear search searchlight seasick season seat second secret sect section sector sedan sediment see seed seek seem seep seesaw segment select selector self selfish sell seller semen semicolon semiconductor seminar send senior sensor sensual sentiment sequel sequin sergeant serial serious sermon serpent serum servant server serviceman session set setback setter settlement settler seven seventeen seventeenth seventh sever sew sewer sex sexual shack shade shadow shaft shaikh shake shallow sham shame shampoo shamrock shape share shark sharp sharpen shatter shave shaver shawl shaykh she sheaf shear sheath shed sheen sheep sheepish sheer sheet sheik sheikh shelf shell shellfish shelter shepherd sherbert sherbet sheriff shield shift shimmer shin shine ship shipment shipwreck shirk shirt shiver shoal shock shoe shoo shoot shop shopper shore short shorten shorthand shot shotgun shoulder shout shove shovel show showdown shower showman shrapnel shred shrew shrewd shriek shrill shrimp shrine shrink shrivel shroud shrub shrug shuck shudder shun shunt shut shutter shyness sic sick sicken side sideshow sidestep sidetrack sidewalk siesta sift sigh sight sign signal signpost silent silicon silk sill silo silt silver silversmith similar simmer sin sinew sing singer singular sink sinner sinus sip siphon sir sire siren sirloin sirup sister sisterhood sit site sitter six sixteen sixteenth sixth sixtieth size skate skateboard skater skein skeleton skeptic sketch skew skewer ski skid skill skillet skim skimp skin skinflint skip skipper skirmish skirt skit skulk skull skunk sky skylight skyrocket slab slack slacken slake slam slander slang slant slap slapstick slash slat slate slaughter slave slay sled sleek sleep sleeper sleet sleigh slender slew slice slick slide slight slim slime sling slingshot slink slip slipper slit slither sliver slob slobber slog slogan slop slope slosh slot sloth slouch slow slug slum slumber slump slur slush slut slyness smack small smallpox smart smash smear smell smelt smidgen smidgeon smidgin smile smirk smite smith smock smog smoke smoker smokestack smolder smooth smother smoulder smug smuggler smut snack snag snail snake snap snapshot snare snarl snatch sneak sneaker sneer snicker sniff snip snipe sniper snippet snitch snob snoop snore snorkel snort snot snout snow snowdrift snowplow snowstorm snub snuff snug so soak soap soar sob sober soccer social socialist sociologist sock socket sod soda sodium sofa soft soften soil sojourn solder soldier sole solemn solicit solicitor solid solo soloist solvent somersault somewhat son sonata song sonnet soot sop soprano sorceress sore sorrow sort soul sound soundproof soup sour south southeast southern southpaw southward southwest souvenir sovereign sow spa space spacecraft spaceship spade spaghetti span spaniel spank spanner spar spare spark sparkler sparrow spars spasm spat spatter spatula spawn spay speak speaker spear spearhead spearmint special specialist specimen speck spectacular specter spectrum speech speed speedboat spell spellbind spend spendthrift sperm spew sphere sphinx spice spider spigot spike spill spin spinach spinal spine spinster spiral spire spirit spit spite spiteful splash splatter spleen splendid splendor splice splint splinter split spoil spoke spokesman spokesperson spokeswoman sponsor spoof spook spool spoon spoons spore sport sportsmanship spot spotlight spout sprain sprawl spray spread spreadsheet spree sprig spring springboard sprinkler sprint sprinter sprout spruce spud spunk spur spurn spurt sputter squad squadron squalid squall squalor squander squash squat squawk squeak squeal squelch squid squint squirm squirrel squirt stab stack stadium staff stag stage stagecoach stagger stain stair stairway stake stale stalk stall stallion stalwart stamina stammer stamp stamped stanch stand standard standoff standpoint stanza stapler star starboard starch stardom stare starfish starlight start starter state statement statesman statesmanship static station statistician status staunch stave stay steak steal stealth steam steel steep steer stem stench stencil step stereo stern stew steward stewardess stick sticker stickler stiff stiffen stigma still stimulus sting stinger stink stint stir stirrup stitch stock stockyard stoke stole stolid stomach stomp stone stool stoop stop stopgap stopper stopwatch store storeroom storey stork storm stout stove stow stowaway straggler straight straighten straightforward straightjacket strain strainer strait straitjacket strand stranger strap stratagem stratum straw stray streak stream streamer street streetcar strength strengthen strenuous stress stretch stretcher strew strict stride strife strike striker string strip stripe strive stroke stroll stroller strong stronghold strum strut stub stubborn stud student studio stuff stump stun stunt stupid stupor stutter style sub subject sublet submit subscript subset subsist subtract suburb suburban subvert subway succeed success successor succinct succor succumb suck sucker suction sudden sue sued suffer suffix sugar suggest suit suitor sulfur sulk sullen sulphur sultan sum summer summit summon summons sun sunburn sunday sundial sundown sunlight sunscreen sunset suntan sunup super superb superior supermarket superscript superstar supervisor supper supplant supplement supplier support suppress sure surf surfboard surgeon surmount surpass surplus surround survey surveyor survivor suspect suspend suspicion sustain swab swagger swallow swamp swan swap swarm swat sway swear sweat sweater sweep sweeper sweet sweeten sweetheart swell swift swig swill swim swindler swine swing swipe swirl swish switch switchboard swivel swoon swoop sword swordfish syllabus symbol symptom synagog synonym syntax syphon syrup system tab tablecloth tablespoon tablespoons tablet tabloid taboo tabu tacit tack taco tact tactic tactless tag tail taillight tailor tailspin taint take takeoff taker talc tale talent talisman talk talker tallow talon tame tamper tan tandem tang tangent tango tank tankard tanker tantrum tap tape taper tar tarantula target tariff tarnish tarpaulin tart tartan tartar task tassel tattoo taunt taurus tavern tax taxi taxicab tea teach teacher teacup teak team teamster teamwork teapot tear teardrop teas teaspoon teat technician tedious tedium tee teem teen teeter teeth teetotal telegram telegraph tell teller temper temperament tempest tempo tempt ten tenant tend tender tendon tendril tenement tenet tenor tens tension tent tenth term terminus terrain terrier terror terrorist test testament tester tetanus tether text textbook textual thank that thatch thaw theater theft their theist theme then theologian theorem theorist therapist there thermal thermostat thesaurus these thick thicken thicket thief thigh thin thing think thinker third thirst thirteen thirteenth thirtieth thong thorn thorough thou thought thoughtless thousand thousandth thrash thread threat threaten three thresh thresher threshold thrift thrill thriller thrive throat throb throne throng throw throwback thrust thud thug thumb thumbtack thump thunder thunderbolt thunderstorm thursday thwart thyme thyroid tiara tick ticket tidbit tide tie tier tiff tiger tight tighten tightwad tile till tilt timber time timer timid tin tinder ting tinker tinsel tint tip tipi tire tired tit titbit titter to toad toadstool toast toaster tobacco toboggan today toddler toe toenail toga toil toilet token toll tomahawk tomato tomb tomboy tomcat tome tomorrow ton tone tong tonic tonight tonsil tool toot tooth toothbrush toothpick top topaz topic torch torment tormentor tornado torpedo torrent torrid torso tortilla toss tot total totalitarian tote totem totter toucan touch touchdown tough toughen tour tourist tournament tourniquet tout tow toward towel tower town toxin toy trace track tract traction tractor trade trademark trader traffic tragic trail trailer train trainer trait traitor tramp tranquil transact transcend transcript transfer transform transgress transient transistor transit transmit transplant transport trap trapezoid trapper trash trauma travel trawl trawler tray tread treason treat treatment tree trek trellis tremor trench trend trespass trial tribe trick trickster trigger trill trillion trim trinket trio trip tripe triplet tripod triumph trivia trivial troll trolley troop trooper trot trough trouser trout trowel truant truce truck true truism trump trumpet trunk trust truth tryout tsar tub tuba tube tuck tuesday tuft tug tuition tulip tumbler tumor tumult tuna tundra tune tuner tunic tunnel turban tureen turf turkey turmoil turn turnip turnout turret turtleneck tusk tutor tuxedo twang tweak tweed tweet twelfth twentieth twig twilight twin twine twirl twist twister twitch twitter two tycoon type typeset typhoid typhoon typhus typist tyrant tzar udder ulcer ultimatum ultraviolet umbrella unblock unburden unclean uncommon under underbrush undercut underdog undergo underground undergrowth underlay underneath underpass undershirt understand undertow underwear underweight underworld undo undress unearth unequal uneven unfair unfasten unfit unfold unfurl unhook unicorn uniform union unison unit unjust unkind unknown unleash unload unlock unman unmask unpack unpopular unravel unread unrest unsay unscrew unseat unsound unveil unwilling unwind unwrap up upbeat upend uphold upholster upkeep uplift upper upright uproar uproot upset upshot upstart uptown upturn upward uranium urban urchin urgent urn use useless user usher usual usurp utensil uterus utilitarian utmost utter vacuum vagabond vagina vagrant valet valid valley valor van vandal vane vanguard vanilla vanish vanquish vapor variant various varnish vase vast vat vault veal vector veer vegetarian vehement veil vein velour velvet vend vender vendor veneer venison venom vent ventriloquist veranda verandah verb verbal verdict vermin vernacular version vertebra vertigo vessel vest vestment vet veteran veterinarian veto vex viaduct vial vicar vice vicious victim victor video vie view viewer viewpoint vigil vigilant vigor villa villain vine vinegar vineyard vinyl viola violent violet violin viper virgin virtual virtuoso virtuous virus visa vise vision visit visitor visor vista visual vital vitamin vivid vizor vocal vocalist vodka void volcano volley volt vomit voodoo vortex vote voter vouch voucher vow vowel vulgar wad wade wafer waft wag wage wager wagon waif wail waist wait waiter waitress waiver wake waken walk walker walkout wall wallet wallop wallow walnut walrus waltz wand wander wane want wanton war ward warden warhead warm warmth warn warp warpath warrant warren warrior wart wash washcloth washer washout washroom wasp wastebasket wasteland watch watchdog watchman watchword water watercolor waterfront watermark watermelon waterproof waterway watt wave wavelength waver wax way waylay weak weaken wealth wean weapon wear weasel weather weaver web wed wedlock wednesday wee weed week weekday weekend weep weigh weight weird weirdo weld welder well welt welter werewolf west western wet whack whale whaler wharf what wheat wheel wheelbarrow wheelchair when where whet whiff while whim whimper whine whip whir whirl whirlpool whirlwind whirr whisk whisker whiskey whisper white whiten whitewash whiz whizz whole wholes whoop whopper whore wick wicked wicker wicket wide widen widow width wield wife wig wigwam wild wildcat wilder will willing willow wilt win winch wind window windscreen windshield wine wing wink winner winter wipe wiper wire wisdom wise wisecrack wish wisp wist wit witch witchcraft withdraw wither withhold within withstand wive wiz wizard woe wok wolf woman womanhood womankind womb wombat won wonder wonderland wont woo wood woodchuck wooden woodland woodsman woodwind woodwork woof wool woolen word work workbench workbook worker workman workmanship workout workshop world worm worsen worship worst worth would wound wow wrangler wrap wrapper wrath wreak wreath wreck wren wrench wrest wrestler wretch wretched wring wringer wrist wristwatch writ write writer wrong wrongdoer xmas yacht yack yak yam yank yap yard yardstick yarn yawn year yearn yeast yell yellow yelp yen yes yesterday yew yiddish yield yock yodel yoga yoghourt yoghurt yogurt yoke yokel yolk you young youngster your youth yowl yuck zeal zebra zenith zero zest zigzag zillion zinc zip zipper zodiac zombi zone zoo zoologist zoom zucchini pyutil-3.3.2/pyutil/dictutil.py000066400000000000000000000501551437014040400165710ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- """ Tools to mess with dicts. """ import warnings import copy, operator from bisect import bisect_left, insort_left from pyutil.assertutil import _assert, precondition def move(k, d1, d2, strict=False): """ Move item with key k from d1 to d2. """ warnings.warn("deprecated", DeprecationWarning) if strict and not d1.has_key(k): raise KeyError(k) d2[k] = d1[k] del d1[k] def subtract(d1, d2): """ Remove all items from d1 whose key occurs in d2. @returns d1 """ warnings.warn("deprecated", DeprecationWarning) if len(d1) > len(d2): for k in d2.keys(): if d1.has_key(k): del d1[k] else: for k in d1.keys(): if d2.has_key(k): del d1[k] return d1 class DictOfSets(dict): def add(self, key, value): warnings.warn("deprecated", DeprecationWarning) if key in self: self[key].add(value) else: self[key] = set([value]) def discard(self, key, value): warnings.warn("deprecated", DeprecationWarning) if not key in self: return self[key].discard(value) if not self[key]: del self[key] class UtilDict: def __init__(self, initialdata={}): warnings.warn("deprecated", DeprecationWarning) self.d = {} self.update(initialdata) def del_if_present(self, key): if self.has_key(key): del self[key] def items_sorted_by_value(self): """ @return a sequence of (key, value,) pairs sorted according to value """ l = [(x[1], x[0],) for x in self.d.items()] l.sort() return [(x[1], x[0],) for x in l] def items_sorted_by_key(self): """ @return a sequence of (key, value,) pairs sorted according to key """ l = self.d.items() l.sort() return l def __repr__(self, *args, **kwargs): return self.d.__repr__(*args, **kwargs) def __str__(self, *args, **kwargs): return self.d.__str__(*args, **kwargs) def __contains__(self, *args, **kwargs): return self.d.__contains__(*args, **kwargs) def __len__(self, *args, **kwargs): return self.d.__len__(*args, **kwargs) def __cmp__(self, other): try: return self.d.__cmp__(other) except TypeError as le: # maybe we should look for a .d member in other. I know this is insanely kludgey, but the Right Way To Do It is for dict.__cmp__ to use structural typing ("duck typing") try: return self.d.__cmp__(other.d) except: raise le def __eq__(self, other): return self.d == other def __ne__(self, other): return self.d != other def __gt__(self, other): return self.d > other def __ge__(self, other): return self.d >= other def __le__(self, other): return self.d < other def __lt__(self, other): return self.d <= other def __getitem__(self, *args, **kwargs): return self.d.__getitem__(*args, **kwargs) def __setitem__(self, *args, **kwargs): return self.d.__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): return self.d.__delitem__(*args, **kwargs) def __iter__(self, *args, **kwargs): return self.d.__iter__(*args, **kwargs) def clear(self, *args, **kwargs): return self.d.clear(*args, **kwargs) def copy(self, *args, **kwargs): return self.__class__(self.d.copy(*args, **kwargs)) def fromkeys(self, *args, **kwargs): return self.__class__(self.d.fromkeys(*args, **kwargs)) def get(self, key, default=None): return self.d.get(key, default) def has_key(self, *args, **kwargs): return self.d.__contains__(*args, **kwargs) def items(self, *args, **kwargs): return self.d.items(*args, **kwargs) if hasattr(dict, 'iteritems'): # PY2 def iteritems(self, *args, **kwargs): return self.d.iteritems(*args, **kwargs) if hasattr(dict, 'iterkeys'): # PY2 def iterkeys(self, *args, **kwargs): return self.d.iterkeys(*args, **kwargs) if hasattr(dict, 'itervalues'): # PY2 def itervalues(self, *args, **kwargs): return self.d.itervalues(*args, **kwargs) def keys(self, *args, **kwargs): return self.d.keys(*args, **kwargs) def pop(self, *args, **kwargs): return self.d.pop(*args, **kwargs) def popitem(self, *args, **kwargs): return self.d.popitem(*args, **kwargs) def setdefault(self, *args, **kwargs): return self.d.setdefault(*args, **kwargs) def update(self, *args, **kwargs): self.d.update(*args, **kwargs) def values(self, *args, **kwargs): return self.d.values(*args, **kwargs) class NumDict: def __init__(self, initialdict={}): warnings.warn("deprecated", DeprecationWarning) self.d = copy.deepcopy(initialdict) def merge(self, otherdict): """ Add all the values from otherdict into this dict. """ for key, val in otherdict.items(): self.add_num(key, val) def add_num(self, key, val, default=0): """ If the key doesn't appear in self then it is created with value default (before addition). """ self.d[key] = self.d.get(key, default) + val def subtract_num(self, key, val, default=0): self.d[key] = self.d.get(key, default) - val def sum(self): """ @return: the sum of all values """ return reduce(operator.__add__, self.d.values()) def inc(self, key, default=0): """ Increment the value associated with key in dict. If there is no such key, then one will be created with initial value 0 (before inc() -- therefore value 1 after inc). """ self.add_num(key, 1, default) def dec(self, key, default=0): """ Decrement the value associated with key in dict. If there is no such key, then one will be created with initial value 0 (before dec() -- therefore value -1 after dec). """ self.subtract_num(key, 1, default) def items_sorted_by_value(self): """ @return a sequence of (key, value,) pairs sorted according to value """ l = [(x[1], x[0],) for x in self.d.items()] l.sort() return [(x[1], x[0],) for x in l] def item_with_largest_value(self): it = iter(self.d.items()) (winner, winnerval,) = next(it) try: while True: n, nv = it.next() if nv > winnerval: winner = n winnerval = nv except StopIteration: pass return (winner, winnerval,) def items_sorted_by_key(self): """ @return a sequence of (key, value,) pairs sorted according to key """ l = self.d.items() l.sort() return l def __repr__(self, *args, **kwargs): return self.d.__repr__(*args, **kwargs) def __str__(self, *args, **kwargs): return self.d.__str__(*args, **kwargs) def __contains__(self, *args, **kwargs): return self.d.__contains__(*args, **kwargs) def __len__(self, *args, **kwargs): return self.d.__len__(*args, **kwargs) def __cmp__(self, other): try: return self.d.__cmp__(other) except TypeError as le: # maybe we should look for a .d member in other. I know this is insanely kludgey, but the Right Way To Do It is for dict.__cmp__ to use structural typing ("duck typing") try: return self.d.__cmp__(other.d) except: raise le def __eq__(self, other): return self.d == other def __ne__(self, other): return self.d != other def __gt__(self, other): return self.d > other def __ge__(self, other): return self.d >= other def __le__(self, other): return self.d < other def __lt__(self, other): return self.d <= other def __getitem__(self, *args, **kwargs): return self.d.__getitem__(*args, **kwargs) def __setitem__(self, *args, **kwargs): return self.d.__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): return self.d.__delitem__(*args, **kwargs) def __iter__(self, *args, **kwargs): return self.d.__iter__(*args, **kwargs) def clear(self, *args, **kwargs): return self.d.clear(*args, **kwargs) def copy(self, *args, **kwargs): return self.__class__(self.d.copy(*args, **kwargs)) def fromkeys(self, *args, **kwargs): return self.__class__(self.d.fromkeys(*args, **kwargs)) def get(self, key, default=0): return self.d.get(key, default) def has_key(self, *args, **kwargs): return self.d.__contains__(*args, **kwargs) def items(self, *args, **kwargs): return self.d.items(*args, **kwargs) if hasattr(dict, 'iteritems'): # PY2 def iteritems(self, *args, **kwargs): return self.d.iteritems(*args, **kwargs) if hasattr(dict, 'iterkeys'): # PY2 def iterkeys(self, *args, **kwargs): return self.d.iterkeys(*args, **kwargs) if hasattr(dict, 'itervalues'): # PY2 def itervalues(self, *args, **kwargs): return self.d.itervalues(*args, **kwargs) def keys(self, *args, **kwargs): return self.d.keys(*args, **kwargs) def pop(self, *args, **kwargs): return self.d.pop(*args, **kwargs) def popitem(self, *args, **kwargs): return self.d.popitem(*args, **kwargs) def setdefault(self, *args, **kwargs): return self.d.setdefault(*args, **kwargs) def update(self, *args, **kwargs): return self.d.update(*args, **kwargs) def values(self, *args, **kwargs): return self.d.values(*args, **kwargs) def del_if_present(d, k): if k in d: del d[k] class ValueOrderedDict: """ Note: this implementation assumes that the values do not mutate and change their sort order. That is, it stores the values in a sorted list and as items are added and removed from the dict, it makes updates to the list which will keep the list sorted. But if a value that is currently sitting in the list changes its sort order, then the internal consistency of this object will be lost. If that happens, and if assertion checking is turned on, then you will get an assertion failure the very next time you try to do anything with this ValueOrderedDict. However, those internal consistency checks are very slow and almost certainly unacceptable to leave turned on in production code. """ class ItemIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def __next__(self): precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c) precondition((self.i == len(self.c.l)) or self.c.l[self.i][1] in self.c.d, "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c) if self.i == len(self.c.l): raise StopIteration() le = self.c.l[self.i] self.i += 1 return (le[1], le[0],) def next(self): return self.__next__() def iteritems(self): return ValueOrderedDict.ItemIterator(self) def items(self): return zip(map(operator.__getitem__, self.l, [1]*len(self.l)), map(operator.__getitem__, self.l, [0]*len(self.l))) def values(self): return map(operator.__getitem__, self.l, [0]*len(self.l)) def keys(self): return map(operator.__getitem__, self.l, [1]*len(self.l)) class KeyIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def __next__(self): precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c) precondition((self.i == len(self.c.l)) or self.c.l[self.i][1] in self.c.d, "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c) if self.i == len(self.c.l): raise StopIteration() le = self.c.l[self.i] self.i += 1 return le[1] def next(self): return self.__next__() def iterkeys(self): return ValueOrderedDict.KeyIterator(self) def __iter__(self): return ValueOrderedDict.KeyIterator(self) class ValueIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def __next__(self): precondition(self.i <= len(self.c.l), "The iterated ValueOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, self.c) precondition((self.i == len(self.c.l)) or self.c.l[self.i][1] in self.c.d, "The iterated ValueOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the ValueOrderedDict while the iteration was in progress.", self.i, (self.i < len(self.c.l)) and self.c.l[self.i], self.c) if self.i == len(self.c.l): raise StopIteration() le = self.c.l[self.i] self.i += 1 return le[0] def next(self): return self.__next__() def itervalues(self): return ValueOrderedDict.ValueIterator(self) def __init__(self, initialdata={}): warnings.warn("deprecated", DeprecationWarning) self.d = {} # k: key, v: val self.l = [] # sorted list of tuples of (val, key,) self.update(initialdata) assert self._assert_invariants() def __len__(self): return len(self.l) def __repr_n__(self, n=None): s = ["{",] try: iter = self.iteritems() x = iter.next() s.append(str(x[0])); s.append(": "); s.append(str(x[1])) i = 1 while (n is None) or (i < n): x = iter.next() s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1])) except StopIteration: pass s.append("}") return ''.join(s) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) def __eq__(self, other): for (k, v,) in other.items(): if k not in self.d or self.d[k] != v: return False return True def __ne__(self, other): return not self.__eq__(other) def _assert_invariants(self): it = iter(self.l) try: oldx = next(it) while True: x = next(it) # self.l is required to be sorted _assert(x >= oldx, x, oldx) # every element of self.l is required to appear in self.d _assert(x[1] in self.d, x) oldx =x except StopIteration: pass for (k, v,) in self.d.items(): i = bisect_left(self.l, (v, k,)) while (self.l[i][0] is not v) or (self.l[i][1] is not k): i += 1 _assert(i < len(self.l), i, len(self.l), k, v, self.l) _assert(self.l[i][0] is v, i, v, l=self.l, d=self.d) _assert(self.l[i][1] is k, i, k, l=self.l, d=self.d) return True def insert(self, key, val=None): assert self._assert_invariants() result = self.__setitem__(key, val) assert self._assert_invariants() return result def setdefault(self, key, default=None): assert self._assert_invariants() if not self.has_key(key): self[key] = default assert self._assert_invariants() return self[key] def __setitem__(self, key, val=None): assert self._assert_invariants() if key in self.d: oldval = self.d[key] if oldval != val: # re-sort i = bisect_left(self.l, (oldval, key,)) while (self.l[i][0] is not oldval) or (self.l[i][1] is not key): i += 1 self.l.pop(i) insort_left(self.l, (val, key,)) elif oldval is not val: # replace i = bisect_left(self.l, (oldval, key,)) while (self.l[i][0] is not oldval) or (self.l[i][1] is not key): i += 1 self.l[i] = (val, key,) else: insort_left(self.l, (val, key,)) self.d[key] = val assert self._assert_invariants() return val def remove(self, key, default=None, strictkey=True): assert self._assert_invariants() result = self.__delitem__(key, default, strictkey) assert self._assert_invariants() return result def __getitem__(self, key, default=None, strictkey=True): if key not in self.d: if strictkey: raise KeyError(key) else: return default return self.d[key] def __delitem__(self, key, default=None, strictkey=True): """ @param strictkey: True if you want a KeyError in the case that key is not there, False if you want a reference to default in the case that key is not there @param default: the object to return if key is not there; This is ignored if strictkey. @return: the object removed or default if there is not item by that key and strictkey is False """ assert self._assert_invariants() if key in self.d: val = self.d.pop(key) i = bisect_left(self.l, (val, key,)) while (self.l[i][0] is not val) or (self.l[i][1] is not key): i += 1 self.l.pop(i) assert self._assert_invariants() return val elif strictkey: assert self._assert_invariants() raise KeyError(key) else: assert self._assert_invariants() return default def clear(self): assert self._assert_invariants() self.d.clear() del self.l[:] assert self._assert_invariants() def update(self, otherdict): """ @return: self """ assert self._assert_invariants() for (k, v,) in otherdict.items(): self.insert(k, v) assert self._assert_invariants() return self def has_key(self, key): assert self._assert_invariants() return key in self.d def popitem(self): if not self.l: raise KeyError('popitem(): dictionary is empty') le = self.l.pop(0) del self.d[le[1]] return (le[1], le[0],) def pop(self, k, default=None, strictkey=False): if k in self.d: if strictkey: raise KeyError(k) else: return default v = self.d.pop(k) i = bisect_left(self.l, (v, k,)) while (self.l[i][0] is not v) or (self.l[i][1] is not k): i += 1 self.l.pop(i) return v def pop_from_list(self, i=0): le = self.l.pop(i) del self.d[le[1]] return le[1] pyutil-3.3.2/pyutil/fileutil.py000066400000000000000000000212531437014040400165620ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ Futz with files like a pro. """ import errno, os, stat, tempfile try: import bsddb except ImportError: DBNoSuchFileError = None else: DBNoSuchFileError = bsddb.db.DBNoSuchFileError # read_file() and write_file() copied from Mark Seaborn's blog post. Please # read it for complete rationale: # http://lackingrhoticity.blogspot.com/2009/12/readfile-and-writefile-in-python.html def read_file(filename, mode='rb'): """ Read the contents of the file named filename and return it in a string. This function closes the file handle before it returns (even if the underlying Python implementation's garbage collector doesn't). """ fh = open(filename, mode) try: return fh.read() finally: fh.close() def write_file(filename, data, mode='wb'): """ Write the string data into a file named filename. This function closes the file handle (ensuring that the written data is flushed from the perspective of the Python implementation) before it returns (even if the underlying Python implementation's garbage collector doesn't).""" fh = open(filename, mode) try: fh.write(data) finally: fh.close() # For backwards-compatibility in case someone is using these names. We used to # have a superkludge in fileutil.py under these names. def rename(src, dst, tries=4, basedelay=0.1): return os.rename(src, dst) def remove(f, tries=4, basedelay=0.1): return os.remove(f) def rmdir(f, tries=4, basedelay=0.1): return os.rmdir(f) class _Dir(object): """ Hold a set of files and subdirs and clean them all up when asked to. """ def __init__(self, name, cleanup=True): self.name = name self.cleanup = cleanup self.files = [] self.subdirs = set() def file(self, fname, mode=None): """ Create a file in the tempdir and remember it so as to close() it before attempting to cleanup the temp dir. @rtype: file """ ffn = os.path.join(self.name, fname) if mode is not None: fo = open(ffn, mode) else: fo = open(ffn) self.register_file(fo) return fo def subdir(self, dirname): """ Create a subdirectory in the tempdir and remember it so as to call shutdown() on it before attempting to clean up. @rtype: _Dir instance """ ffn = os.path.join(self.name, dirname) sd = _Dir(ffn, self.cleanup) self.register_subdir(sd) make_dirs(sd.name) return sd def register_file(self, fileobj): """ Remember the file object and call close() on it before attempting to clean up. """ self.files.append(fileobj) def register_subdir(self, dirobj): """ Remember the _Dir object and call shutdown() on it before attempting to clean up. """ self.subdirs.add(dirobj) def shutdown(self): if self.cleanup: for subdir in hasattr(self, 'subdirs') and self.subdirs or []: subdir.shutdown() for fileobj in hasattr(self, 'files') and self.files or []: if DBNoSuchFileError is None: fileobj.close() # "close()" is idempotent so we don't need to catch exceptions here else: try: fileobj.close() except DBNoSuchFileError: # Ah, except that the bsddb module's file-like object (a DB object) has a non-idempotent close... pass if hasattr(self, 'name'): rm_dir(self.name) def __repr__(self): return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) def __str__(self): return self.__repr__() def __del__(self): try: self.shutdown() except: import traceback traceback.print_exc() class NamedTemporaryDirectory(_Dir): """ Call tempfile.mkdtemp(), store the name of the dir in self.name, and rm_dir() when it gets garbage collected or "shutdown()". Also keep track of file objects for files within the tempdir and call close() on them before rm_dir(). This is a convenient way to open temp files within the directory, and it is very helpful on Windows because you can't delete a directory which contains a file which is currently open. """ def __init__(self, cleanup=True, *args, **kwargs): """ If cleanup, then the directory will be rmrf'ed when the object is shutdown. """ name = tempfile.mkdtemp(*args, **kwargs) _Dir.__init__(self, name, cleanup) class ReopenableNamedTemporaryFile: """ This uses tempfile.mkstemp() to generate a secure temp file. It then closes the file, leaving a zero-length file as a placeholder. You can get the filename with ReopenableNamedTemporaryFile.name. When the ReopenableNamedTemporaryFile instance is garbage collected or its shutdown() method is called, it deletes the file. """ def __init__(self, *args, **kwargs): fd, self.name = tempfile.mkstemp(*args, **kwargs) os.close(fd) def __repr__(self): return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name) def __str__(self): return self.__repr__() def __del__(self): self.shutdown() def shutdown(self): remove(self.name) def make_dirs(dirname, mode=0o777): """ An idempotent version of os.makedirs(). If the dir already exists, do nothing and return without raising an exception. If this call creates the dir, return without raising an exception. If there is an error that prevents creation or if the directory gets deleted after make_dirs() creates it and before make_dirs() checks that it exists, raise an exception. """ tx = None try: os.makedirs(dirname, mode) except OSError as x: tx = x if not os.path.isdir(dirname): if tx: raise tx raise IOError("unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname) # careful not to construct an IOError with a 2-tuple, as that has a special meaning... def rmtree(dirname): """ A threadsafe and idempotent version of shutil.rmtree(). If the dir is already gone, do nothing and return without raising an exception. If this call removes the dir, return without raising an exception. If there is an error that prevents deletion or if the directory gets created again after rm_dir() deletes it and before rm_dir() checks that it is gone, raise an exception. """ excs = [] try: os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) for f in os.listdir(dirname): fullname = os.path.join(dirname, f) if os.path.isdir(fullname): rm_dir(fullname) else: remove(fullname) os.rmdir(dirname) except EnvironmentError as le: # Ignore "No such file or directory", collect any other exception. if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): excs.append(le) except Exception as le: excs.append(le) # Okay, now we've recursively removed everything, ignoring any "No # such file or directory" errors, and collecting any other errors. if os.path.exists(dirname): if len(excs) == 1: raise excs[0] if len(excs) == 0: raise OSError("Failed to remove dir for unknown reason.") raise OSError(excs) def rm_dir(dirname): # Renamed to be like shutil.rmtree and unlike rmdir. return rmtree(dirname) def remove_if_possible(f): try: remove(f) except EnvironmentError: pass def remove_if_present(f): try: remove(f) except EnvironmentError as le: # Ignore "No such file or directory", re-raise any other exception. if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT): raise def rmdir_if_possible(f): try: rmdir(f) except EnvironmentError: pass def open_or_create(fname, binarymode=True): try: f = open(fname, binarymode and "r+b" or "r+") except EnvironmentError: f = open(fname, binarymode and "w+b" or "w+") return f def du(basedir): size = 0 for root, dirs, files in os.walk(basedir): for f in files: fn = os.path.join(root, f) size += os.path.getsize(fn) return size pyutil-3.3.2/pyutil/find_exe.py000066400000000000000000000017371437014040400165330ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- import warnings import os, sys from twisted.python.procutils import which def find_exe(exename): """ Look for something named exename or exename + ".py". This is a kludge. @return: a list containing one element which is the path to the exename (if it is thought to be executable), or else the first element being sys.executable and the second element being the path to the exename + ".py", or else return False if one can't be found """ warnings.warn("deprecated", DeprecationWarning) exes = which(exename) exe = exes and exes[0] if not exe: exe = os.path.join(sys.prefix, 'scripts', exename + '.py') if os.path.exists(exe): path, ext = os.path.splitext(exe) if ext.lower() in [".exe", ".bat",]: cmd = [exe,] else: cmd = [sys.executable, exe,] return cmd else: return False pyutil-3.3.2/pyutil/humanreadable.py000066400000000000000000000106541437014040400175400ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import os import itertools try: from reprlib import Repr except ImportError: from repr import Repr class BetterRepr(Repr): def __init__(self): Repr.__init__(self) # Note: These levels can get adjusted dynamically! My goal is to get more info when printing important debug stuff like exceptions and stack traces and less info when logging normal events. --Zooko 2000-10-14 self.maxlevel = 6 self.maxdict = 6 self.maxlist = 6 self.maxtuple = 6 self.maxstring = 300 self.maxother = 300 def repr_function(self, obj, level): if hasattr(obj, 'func_code'): return '<' + obj.func_name + '() at ' + os.path.basename(obj.func_code.co_filename) + ':' + str(obj.func_code.co_firstlineno) + '>' else: return '<' + obj.func_name + '() at (builtin)' def repr_instance_method(self, obj, level): if hasattr(obj, 'func_code'): return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at ' + os.path.basename(obj.im_func.func_code.co_filename) + ':' + str(obj.im_func.func_code.co_firstlineno) + '>' else: return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at (builtin)' def repr_long(self, obj, level): s = repr(obj) # XXX Hope this isn't too slow... if len(s) > self.maxlong: i = max(0, (self.maxlong-3)/2) j = max(0, self.maxlong-3-i) s = s[:i] + '...' + s[len(s)-j:] if s[-1] == 'L': return s[:-1] return s def repr_instance(self, obj, level): """ If it is an instance of Exception, format it nicely (trying to emulate the format that you see when an exception is actually raised, plus bracketing '<''s). If it is an instance of dict call self.repr_dict() on it. If it is an instance of list call self.repr_list() on it. Else call Repr.repr_instance(). """ if isinstance(obj, Exception): # Don't cut down exception strings so much. tms = self.maxstring self.maxstring = max(512, tms * 4) tml = self.maxlist self.maxlist = max(12, tml * 4) try: if hasattr(obj, 'args'): if len(obj.args) == 1: return '<' + obj.__class__.__name__ + ': ' + self.repr1(obj.args[0], level-1) + '>' else: return '<' + obj.__class__.__name__ + ': ' + self.repr1(obj.args, level-1) + '>' else: return '<' + obj.__class__.__name__ + '>' finally: self.maxstring = tms self.maxlist = tml if isinstance(obj, dict): return self.repr_dict(obj, level) if isinstance(obj, list): return self.repr_list(obj, level) return Repr.repr_instance(self, obj, level) def repr_list(self, obj, level): """ copied from standard repr.py and fixed to work on multithreadedly mutating lists. """ if level <= 0: return '[...]' n = len(obj) myl = obj[:min(n, self.maxlist)] s = '' for item in myl: entry = self.repr1(item, level-1) if s: s = s + ', ' s = s + entry if n > self.maxlist: s = s + ', ...' return '[' + s + ']' def repr_dict(self, obj, level): """ copied from standard repr.py and fixed to work on multithreadedly mutating dicts. """ if level <= 0: return '{...}' s = '' n = len(obj) items = list(itertools.islice(obj.items(), self.maxdict)) items.sort() for key, val in items: entry = self.repr1(key, level-1) + ':' + self.repr1(val, level-1) if s: s = s + ', ' s = s + entry if n > self.maxdict: s = s + ', ...' return '{' + s + '}' # This object can be changed by other code updating this module's "brepr" # variables. This is so that (a) code can use humanreadable with # "from humanreadable import hr; hr(mything)", and (b) code can override # humanreadable to provide application-specific human readable output # (e.g. libbase32's base32id.AbbrevRepr). brepr = BetterRepr() def hr(x): return brepr.repr(x) pyutil-3.3.2/pyutil/increasing_timer.py000066400000000000000000000146701437014040400202740ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ This module was invented when it was discovered that time.time() can return decreasing answers, which was causing scheduled tasks to get executed out of order. See python bug report `[ #447945 ] time.time() is not non-decreasing', http://sourceforge.net/tracker/index.php?func=detail&aid=447945&group_id=5470&atid=105470 http://mail.python.org/pipermail/python-list/2001-August/thread.html#58296 After posting that bug report, I figured out that this isn't really a bug, but a misunderstanding about the semantics of gettimeofday(). gettimeofday() relies on the hardware clock, which is supposed to reflect the "real" time i.e. the position and orientation of our planet with regard to our sun. But the hardware clock gets adjusted, either for skew (because hardware clocks always run a little faster or a little slower than they ought), or in order to sync up with another clock e.g. through NTP. So it isn't really a bug in the underlying platform (except perhaps a bug in the lack of a prominent warning in the documentation), but if you depend on a monotonically increasing timestamps, you need to use IncreasingTimer.time() instead of the Python standard library's time.time(). --Zooko 2001-08-04 """ import time as standardtime # Here is a global reference to an IncreasingTimer. # This singleton global IncreasingTimer instance gets created at module load time. timer = None class IncreasingTimer: def __init__(self, inittime=None): """ @param inittime starting time (in seconds) or None in which case it will be initialized to standardtime.time() """ if inittime is None: inittime = standardtime.time() self.lasttime = inittime # This stores the most recent answer that we returned from time(). self.delta = 0 # We add this to the result from the underlying standardtime.time(). # How big of an increment do we need to add in order to make the new float greater than the old float? trye = 1.0 while (self.lasttime + trye) > self.lasttime: olde = trye trye = trye / 2.0 self._EPSILON = olde def time(self): """ This returns the current time as a float, with as much precision as the underlying Python interpreter can muster. In addition, successive calls to time() always return bigger numbers. (standardtime.time() can sometimes return the same or even a *smaller* number!) On the other hand, calling time() is a bit slower than calling standardtime.time(), so you might want to avoid it inside tight loops and deal with decreasing or identical answers yourself. Now by definition you cannot "reset" this clock to an earlier state. This means that if you start a Python interpreter and instantiate an IncreasingTimer, and then you subsequently realize that your computer's clock was set to next year, and you set it back to the correct year, that subsequent calls to standardtime.time() will return a number indicating this year and IncreasingTimer.time() will continue to return a number indicating next year. Therefore, you should use the answers from IncreasingTimer.time() in such a way that the only things you depend on are correctness in the relative *order* of two times, (and, with the following caveat, the relative *difference* between two times as well), not the global "correctness" of the times with respect to the rest of the world. The caveat is that if the underlying answers from standardtime.time() jump *forward*, then this *does* distort the relative difference between two answers from IncreasingTimer.time(). What IncreasingTimer.time() does is if the underlying clock goes *backwards*, then IncreasingTimer.time() still returns successively higher numbers. Then if the underlying clock jumps *forwards*, IncreasingTimer.time() also jumps forward the same amount. A weird consequence of this is that if you were to set your system clock to point to 10 years ago, and call: t1 = increasingtimer.time() and then set your system clock back to the present, and call: t2 = increasingtimer.time() , then there would be a 10-year difference between t2 and t1. In practice, adjustments to the underlying system time are rarely that drastic, and for some systems (e.g. Mnet's DoQ, for which this module was invented) it doesn't matter anyway if time jumps forward. Another note: Brian Warner has pointed out that there is another caveat, which is due to there being a delay between successive calls to IncreasingTimer.time(). When the underlying clock jumps backward, then events which were scheduled before the jump and scheduled to go off after the jump may be delayed by at most d, where d is the delay between the two successive calls to IncreasingTimer which spanned the jump. @singlethreaded You must guarantee that you never have more than one thread in this function at a time. """ t = standardtime.time() + self.delta lasttime = self.lasttime if t <= lasttime: self.delta = self.delta + (lasttime - t) + self._EPSILON t = lasttime + self._EPSILON # TODO: If you were sure that you could generate a bigger float in one # pass, you could change this `while' to an `if' and optimize out a # test. while t <= lasttime: # We can get into here only if self._EPSILON is too small to make # # the time float "tick over" to a new higher value. So we # (permanently) # double self._EPSILON. # TODO: Is doubling epsilon the best way to quickly get a # minimally bigger float? self._EPSILON = self._EPSILON * 2.0 # Delta, having smaller magnitude than t, can be incremented by # more than t was incremented. (Up to the old epsilon more.) # That's OK. self.delta = self.delta + self._EPSILON t = t + self._EPSILON self.lasttime = t return t # create the global IncreasingTimer instance and `time' function timer = IncreasingTimer() time = timer.time pyutil-3.3.2/pyutil/iputil.py000066400000000000000000000231411437014040400162510ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # from the Python Standard Library import os, re, socket, sys, subprocess # from Twisted from twisted.internet import defer, threads, reactor from twisted.internet.protocol import DatagramProtocol from twisted.python.procutils import which from twisted.python import log try: import resource def increase_rlimits(): # We'd like to raise our soft resource.RLIMIT_NOFILE, since certain # systems (OS-X, probably solaris) start with a relatively low limit # (256), and some unit tests want to open up more sockets than this. # Most linux systems start with both hard and soft limits at 1024, # which is plenty. # unfortunately the values to pass to setrlimit() vary widely from # one system to another. OS-X reports (256, HUGE), but the real hard # limit is 10240, and accepts (-1,-1) to mean raise it to the # maximum. Cygwin reports (256, -1), then ignores a request of # (-1,-1): instead you have to guess at the hard limit (it appears to # be 3200), so using (3200,-1) seems to work. Linux reports a # sensible (1024,1024), then rejects (-1,-1) as trying to raise the # maximum limit, so you could set it to (1024,1024) but you might as # well leave it alone. try: current = resource.getrlimit(resource.RLIMIT_NOFILE) except AttributeError: # we're probably missing RLIMIT_NOFILE return if current[0] >= 1024: # good enough, leave it alone return try: if current[1] > 0 and current[1] < 1000000: # solaris reports (256, 65536) resource.setrlimit(resource.RLIMIT_NOFILE, (current[1], current[1])) else: # this one works on OS-X (bsd), and gives us 10240, but # it doesn't work on linux (on which both the hard and # soft limits are set to 1024 by default). resource.setrlimit(resource.RLIMIT_NOFILE, (-1,-1)) new = resource.getrlimit(resource.RLIMIT_NOFILE) if new[0] == current[0]: # probably cygwin, which ignores -1. Use a real value. resource.setrlimit(resource.RLIMIT_NOFILE, (3200,-1)) except ValueError: log.msg("unable to set RLIMIT_NOFILE: current value %s" % (resource.getrlimit(resource.RLIMIT_NOFILE),)) except: # who knows what. It isn't very important, so log it and continue log.err() except ImportError: def _increase_rlimits(): # TODO: implement this for Windows. Although I suspect the # solution might be "be running under the iocp reactor and # make this function be a no-op". pass # pyflakes complains about two 'def FOO' statements in the same time, # since one might be shadowing the other. This hack appeases pyflakes. increase_rlimits = _increase_rlimits def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET """ Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad strings) that are currently configured on this host, sorted in descending order of how likely we think they are to work. @param target: we want to learn an IP address they could try using to connect to us; The default value is fine, but it might help if you pass the address of a host that you are actually trying to be reachable to. """ addresses = [] local_ip = get_local_ip_for(target) if local_ip: addresses.append(local_ip) if sys.platform == "cygwin": d = _cygwin_hack_find_addresses(target) else: d = _find_addresses_via_config() def _collect(res): for addr in res: if addr != "0.0.0.0" and not addr in addresses: addresses.append(addr) return addresses d.addCallback(_collect) return d def get_local_ip_for(target): """Find out what our IP address is for use by a given target. @return: the IP address as a dotted-quad string which could be used by to connect to us. It might work for them, it might not. If there is no suitable address (perhaps we don't currently have an externally-visible interface), this will return None. """ try: target_ipaddr = socket.gethostbyname(target) except socket.gaierror: # DNS isn't running, or somehow we encountered an error # note: if an interface is configured and up, but nothing is # connected to it, gethostbyname("A.ROOT-SERVERS.NET") will take 20 # seconds to raise socket.gaierror . This is synchronous and occurs # for each node being started, so users of # test.common.SystemTestMixin (like test_system) will see something # like 120s of delay, which may be enough to hit the default trial # timeouts. For that reason, get_local_addresses_async() was changed # to default to the numerical ip address for A.ROOT-SERVERS.NET, to # avoid this DNS lookup. This also makes node startup fractionally # faster. return None udpprot = DatagramProtocol() port = reactor.listenUDP(0, udpprot) try: udpprot.transport.connect(target_ipaddr, 7) localip = udpprot.transport.getHost().host except socket.error: # no route to that host localip = None port.stopListening() # note, this returns a Deferred return localip # k: result of sys.platform, v: which kind of IP configuration reader we use _platform_map = { "linux-i386": "linux", # redhat "linux-ppc": "linux", # redhat "linux2": "linux", # debian "linux3": "linux", # debian "win32": "win32", "irix6-n32": "irix", "irix6-n64": "irix", "irix6": "irix", "openbsd2": "bsd", "openbsd3": "bsd", "openbsd4": "bsd", "openbsd5": "bsd", "darwin": "bsd", # Mac OS X "freebsd4": "bsd", "freebsd5": "bsd", "freebsd6": "bsd", "freebsd7": "bsd", "freebsd8": "bsd", "freebsd9": "bsd", "netbsd1": "bsd", "netbsd2": "bsd", "netbsd3": "bsd", "netbsd4": "bsd", "netbsd5": "bsd", "netbsd6": "bsd", "dragonfly2": "bsd", "sunos5": "sunos", "cygwin": "cygwin", } class UnsupportedPlatformError(Exception): pass # Wow, I'm really amazed at home much mileage we've gotten out of calling # the external route.exe program on windows... It appears to work on all # versions so far. Still, the real system calls would much be preferred... # ... thus wrote Greg Smith in time immemorial... _win32_path = 'route.exe' _win32_args = ('print',) _win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P
\d+\.\d+\.\d+\.\d+)\s+(?P\d+)\s*$', flags=re.M|re.I|re.S) # These work in Redhat 6.x and Debian 2.2 potato _linux_path = '/sbin/ifconfig' _linux_re = re.compile('^\s*inet [a-zA-Z]*:?(?P
\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) # NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X _netbsd_path = '/sbin/ifconfig' _netbsd_args = ('-a',) _netbsd_re = re.compile('^\s+inet [a-zA-Z]*:?(?P
\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S) # Irix 6.5 _irix_path = '/usr/etc/ifconfig' # Solaris 2.x _sunos_path = '/usr/sbin/ifconfig' # k: platform string as provided in the value of _platform_map # v: tuple of (path_to_tool, args, regex,) _tool_map = { "linux": (_linux_path, (), _linux_re,), "win32": (_win32_path, _win32_args, _win32_re,), "cygwin": (_win32_path, _win32_args, _win32_re,), "bsd": (_netbsd_path, _netbsd_args, _netbsd_re,), "irix": (_irix_path, _netbsd_args, _netbsd_re,), "sunos": (_sunos_path, _netbsd_args, _netbsd_re,), } def _find_addresses_via_config(): return threads.deferToThread(_synchronously_find_addresses_via_config) def _synchronously_find_addresses_via_config(): # originally by Greg Smith, hacked by Zooko to conform to Brian's API platform = _platform_map.get(sys.platform) if not platform: raise UnsupportedPlatformError(sys.platform) (pathtotool, args, regex,) = _tool_map[platform] # If pathtotool is a fully qualified path then we just try that. # If it is merely an executable name then we use Twisted's # "which()" utility and try each executable in turn until one # gives us something that resembles a dotted-quad IPv4 address. if os.path.isabs(pathtotool): return _query(pathtotool, args, regex) else: exes_to_try = which(pathtotool) for exe in exes_to_try: try: addresses = _query(exe, args, regex) except Exception: addresses = [] if addresses: return addresses return [] def _query(path, args, regex): env = {'LANG': 'en_US.UTF-8'} p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) (output, err) = p.communicate() addresses = [] outputsplit = output.split('\n') for outline in outputsplit: m = regex.match(outline) if m: addr = m.groupdict()['address'] if addr not in addresses: addresses.append(addr) return addresses def _cygwin_hack_find_addresses(target): addresses = [] for h in [target, "localhost", "127.0.0.1",]: try: addr = get_local_ip_for(h) if addr not in addresses: addresses.append(addr) except socket.gaierror: pass return defer.succeed(addresses) pyutil-3.3.2/pyutil/jsonutil.py000066400000000000000000000437741437014040400166300ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. # We require simplejson>= 2.1.0 and set its default behavior to # use_decimal=True. This retains backwards compatibility with previous # versions of jsonutil (although it means jsonutil now requires a recent # version of simplejson). # http://code.google.com/p/simplejson/issues/detail?id=34 r"""JSON (JavaScript Object Notation) is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. :mod:`simplejson` exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. It is the externally maintained version of the :mod:`json` library contained in Python 2.6, but maintains compatibility with Python 2.4 and Python 2.5 and (currently) has significant performance advantages, even without using the optional C extension for speedups. Encoding basic Python object hierarchies:: >>> import simplejson as json >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) '["foo", {"bar": ["baz", null, 1.0, 2]}]' >>> print json.dumps("\"foo\bar") "\"foo\bar" >>> print json.dumps(u'\u1234') "\u1234" >>> print json.dumps('\\') "\\" >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) {"a": 0, "b": 0, "c": 0} >>> from StringIO import StringIO >>> io = StringIO() >>> json.dump(['streaming API'], io) >>> io.getvalue() '["streaming API"]' Compact encoding:: >>> import simplejson as json >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) '[1,2,3,{"4":5,"6":7}]' Pretty printing:: >>> import simplejson as json >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) { "4": 5, "6": 7 } Decoding JSON:: >>> import simplejson as json >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj True >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' True >>> from StringIO import StringIO >>> io = StringIO('["streaming API"]') >>> json.load(io)[0] == 'streaming API' True Specializing JSON object decoding:: >>> import simplejson as json >>> def as_complex(dct): ... if '__complex__' in dct: ... return complex(dct['real'], dct['imag']) ... return dct ... >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', ... object_hook=as_complex) (1+2j) >>> from decimal import Decimal >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') True Specializing JSON object encoding:: >>> import simplejson as json >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] ... raise TypeError(repr(o) + " is not JSON serializable") ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) '[2.0, 1.0]' >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) '[2.0, 1.0]' Using simplejson.tool from the shell to validate and pretty-print:: $ echo '{"json":"obj"}' | python -m simplejson.tool { "json": "obj" } $ echo '{ 1.2:3.4}' | python -m simplejson.tool Expecting property name: line 1 column 2 (char 2) """ import pkg_resources pkg_resources.require("simplejson>=2.1.0") # Now we just import all of the contents of the simplejson package and # then overwrite it with a copy of the simplejson __init__.py edited # to make use_decimal=True the default. import simplejson __version__ = simplejson.__version__ __all__ = simplejson.__all__ # The unit tests rely on .encoder and .decoder, and although they are not # included in simplejson.__all__ they are still attributes of the simplejson # package since they are modules within it. from simplejson import encoder, decoder, scanner encoder, decoder, scanner # http://divmod.org/trac/ticket/1499 __all__.extend(['encoder', 'decoder', 'scanner']) __author__ = simplejson.__author__ del simplejson from decimal import Decimal from simplejson.decoder import JSONDecoder, JSONDecodeError JSONDecoder, JSONDecodeError # http://divmod.org/trac/ticket/1499 from simplejson.encoder import JSONEncoder def _import_OrderedDict(): from pyutil.odict import OrderedDict return OrderedDict OrderedDict = _import_OrderedDict() def _import_c_make_encoder(): from simplejson._speedups import make_encoder# XXX try: return make_encoder except ImportError: return None _default_encoder = JSONEncoder( skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, separators=None, encoding='utf-8', default=None, use_decimal=True, ) def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, encoding='utf-8', default=None, use_decimal=True, **kw): """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a ``.write()``-supporting file-like object). If ``skipkeys`` is true then ``dict`` keys that are not basic types (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the some chunks written to ``fp`` may be ``unicode`` instances, subject to normal Python ``str`` to ``unicode`` coercion rules. Unless ``fp.write()`` explicitly understands ``unicode`` (as in ``codecs.getwriter()``) this is likely to cause an error. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If *indent* is a string, then JSON array elements and object members will be pretty-printed with a newline followed by that string repeated for each level of nesting. ``None`` (the default) selects the most compact representation without any newlines. For backwards compatibility with versions of simplejson earlier than 2.1.0, an integer is also accepted and is converted to a string with that many spaces. If ``separators`` is an ``(item_separator, dict_separator)`` tuple then it will be used instead of the default ``(', ', ': ')`` separators. ``(',', ':')`` is the most compact JSON representation. ``encoding`` is the character encoding for str instances, default is UTF-8. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *use_decimal* is true (default: ``True``) then decimal.Decimal will be natively serialized to JSON with full precision. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg. """ # cached encoder if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and encoding == 'utf-8' and default is None and not kw): iterable = _default_encoder.iterencode(obj) else: if cls is None: cls = JSONEncoder iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, encoding=encoding, default=default, use_decimal=use_decimal, **kw).iterencode(obj) # could accelerate with writelines in some versions of Python, at # a debuggability cost for chunk in iterable: fp.write(chunk) def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, encoding='utf-8', default=None, use_decimal=True, **kw): """Serialize ``obj`` to a JSON formatted ``str``. If ``skipkeys`` is false then ``dict`` keys that are not basic types (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the return value will be a ``unicode`` instance subject to normal Python ``str`` to ``unicode`` coercion rules instead of being escaped to an ASCII ``str``. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will result in an ``OverflowError`` (or worse). If ``allow_nan`` is false, then it will be a ``ValueError`` to serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in strict compliance of the JSON specification, instead of using the JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). If ``indent`` is a string, then JSON array elements and object members will be pretty-printed with a newline followed by that string repeated for each level of nesting. ``None`` (the default) selects the most compact representation without any newlines. For backwards compatibility with versions of simplejson earlier than 2.1.0, an integer is also accepted and is converted to a string with that many spaces. If ``separators`` is an ``(item_separator, dict_separator)`` tuple then it will be used instead of the default ``(', ', ': ')`` separators. ``(',', ':')`` is the most compact JSON representation. ``encoding`` is the character encoding for str instances, default is UTF-8. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. If *use_decimal* is true (default: ``True``) then decimal.Decimal will be natively serialized to JSON with full precision. To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the ``.default()`` method to serialize additional types), specify it with the ``cls`` kwarg. """ # cached encoder if (not skipkeys and ensure_ascii and check_circular and allow_nan and cls is None and indent is None and separators is None and encoding == 'utf-8' and default is None and use_decimal and not kw): return _default_encoder.encode(obj) if cls is None: cls = JSONEncoder return cls( skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, allow_nan=allow_nan, indent=indent, separators=separators, encoding=encoding, default=default, use_decimal=use_decimal, **kw).encode(obj) _default_decoder = JSONDecoder(encoding=None, object_hook=None, object_pairs_hook=None, parse_float=Decimal) def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, use_decimal=True, **kw): """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing a JSON document) to a Python object. *encoding* determines the encoding used to interpret any :class:`str` objects decoded by this instance (``'utf-8'`` by default). It has no effect when decoding :class:`unicode` objects. Note that currently only encodings that are a superset of ASCII work, strings of other encodings should be passed in as :class:`unicode`. *object_hook*, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given :class:`dict`. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). *object_pairs_hook* is an optional function that will be called with the result of any object literal decode with an ordered list of pairs. The return value of *object_pairs_hook* will be used instead of the :class:`dict`. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, :func:`collections.OrderedDict` will remember the order of insertion). If *object_hook* is also defined, the *object_pairs_hook* takes priority. *parse_float*, if specified, will be called with the string of every JSON float to be decoded. By default, this is equivalent to ``float(num_str)``. This can be used to use another datatype or parser for JSON floats (e.g. :class:`decimal.Decimal`). *parse_int*, if specified, will be called with the string of every JSON int to be decoded. By default, this is equivalent to ``int(num_str)``. This can be used to use another datatype or parser for JSON integers (e.g. :class:`float`). *parse_constant*, if specified, will be called with one of the following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be used to raise an exception if invalid JSON numbers are encountered. If *use_decimal* is true (default: ``True``) then it implies parse_float=decimal.Decimal for parity with ``dump``. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg. """ return loads(fp.read(), encoding=encoding, cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, use_decimal=use_decimal, **kw) def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, use_decimal=True, **kw): """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) to a Python object. *encoding* determines the encoding used to interpret any :class:`str` objects decoded by this instance (``'utf-8'`` by default). It has no effect when decoding :class:`unicode` objects. Note that currently only encodings that are a superset of ASCII work, strings of other encodings should be passed in as :class:`unicode`. *object_hook*, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given :class:`dict`. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). *object_pairs_hook* is an optional function that will be called with the result of any object literal decode with an ordered list of pairs. The return value of *object_pairs_hook* will be used instead of the :class:`dict`. This feature can be used to implement custom decoders that rely on the order that the key and value pairs are decoded (for example, :func:`collections.OrderedDict` will remember the order of insertion). If *object_hook* is also defined, the *object_pairs_hook* takes priority. *parse_float*, if specified, will be called with the string of every JSON float to be decoded. By default, this is equivalent to ``float(num_str)``. This can be used to use another datatype or parser for JSON floats (e.g. :class:`decimal.Decimal`). *parse_int*, if specified, will be called with the string of every JSON int to be decoded. By default, this is equivalent to ``int(num_str)``. This can be used to use another datatype or parser for JSON integers (e.g. :class:`float`). *parse_constant*, if specified, will be called with one of the following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This can be used to raise an exception if invalid JSON numbers are encountered. If *use_decimal* is true (default: ``True``) then it implies parse_float=decimal.Decimal for parity with ``dump``. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg. """ if (cls is None and encoding is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and use_decimal and not kw): return _default_decoder.decode(s) if cls is None: cls = JSONDecoder if object_hook is not None: kw['object_hook'] = object_hook if object_pairs_hook is not None: kw['object_pairs_hook'] = object_pairs_hook if parse_float is not None: kw['parse_float'] = parse_float if parse_int is not None: kw['parse_int'] = parse_int if parse_constant is not None: kw['parse_constant'] = parse_constant if not use_decimal: kw['use_decimal'] = use_decimal return cls(encoding=encoding, **kw).decode(s) def _toggle_speedups(enabled): import simplejson.decoder as dec import simplejson.encoder as enc import simplejson.scanner as scan c_make_encoder = _import_c_make_encoder() if enabled: dec.scanstring = dec.c_scanstring or dec.py_scanstring enc.c_make_encoder = c_make_encoder enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or enc.py_encode_basestring_ascii) scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner else: dec.scanstring = dec.py_scanstring enc.c_make_encoder = None enc.encode_basestring_ascii = enc.py_encode_basestring_ascii scan.make_scanner = scan.py_make_scanner dec.make_scanner = scan.make_scanner global _default_decoder _default_decoder = JSONDecoder( encoding=None, object_hook=None, object_pairs_hook=None, use_decimal=True, ) global _default_encoder _default_encoder = JSONEncoder( skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, indent=None, separators=None, encoding='utf-8', default=None, use_decimal=True, ) pyutil-3.3.2/pyutil/lineutil.py000077500000000000000000000033321437014040400165730ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- import os, re def lineify_fileobjs(ifo, ofo, strip=False): from pyutil.strutil import pop_trailing_newlines, split_on_newlines for l in ifo: for sl in split_on_newlines(pop_trailing_newlines(l)): if strip: sl = sl.strip() ofo.write(pop_trailing_newlines(sl) + '\n') def lineify_file(fname, strip=False, nobak=True): f = open(fname, "rU") from pyutil.fileutil import ReopenableNamedTemporaryFile rntf = ReopenableNamedTemporaryFile() fo = open(rntf.name, "wb") for l in f: if strip: l = l.strip() + '\n' fo.write(l) fo.close() import shutil if not nobak: shutil.copyfile(fname, fname + ".lines.py-bak") import shutil try: shutil.move(rntf.name, fname) except EnvironmentError: # Couldn't atomically overwrite, so just hope that this process doesn't die # and the target file doesn't get recreated in between the following two # operations: if nobak: os.remove(fname) else: shutil.move(fname, fname + ".lines.py-bak-2") shutil.move(rntf.name, fname) def darcs_metadir_dirpruner(dirs): if "_darcs" in dirs: dirs.remove("_darcs") SCRE=re.compile("\\.(py|php|c|h|cpp|hpp|txt|sh|pyx|pxi|html|htm)$|makefile$", re.IGNORECASE) def source_code_filepruner(fname): return SCRE.search(fname) def all_filepruner(fname): return True def all_dirpruner(dirs): return def lineify_all_files(dirname, strip=False, nobak=True, dirpruner=all_dirpruner, filepruner=all_filepruner): for (root, dirs, files,) in os.walk(dirname): dirpruner(dirs) for fname in files: fullfname = os.path.join(root, fname) if filepruner(fullfname): lineify_file(fullfname, strip=strip, nobak=nobak) pyutil-3.3.2/pyutil/logutil.py000066400000000000000000000011411437014040400164160ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. # This little file makes it so that we can use "log.msg()" and the contents # get logged to the Twisted logger if present, else to the Python Standard # Library logger. import warnings warnings.warn("deprecated", DeprecationWarning) try: from twisted.python import log log # http://divmod.org/trac/ticket/1499 except ImportError: import logging class MinimalLogger: def msg(self, m): logging.log(0, m) log = MinimalLogger() pyutil-3.3.2/pyutil/mathutil.py000066400000000000000000000043541437014040400165770ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ A few commonly needed functions. """ import math def div_ceil(n, d): """ The smallest integer k such that k*d >= n. """ return int((n//d) + (n%d != 0)) def next_multiple(n, k): """ The smallest multiple of k which is >= n. Note that if n is 0 then the answer is 0. """ return div_ceil(n, k) * k def pad_size(n, k): """ The smallest number that has to be added to n to equal a multiple of k. """ if n%k: return k - n%k else: return 0 def is_power_of_k(n, k): return k**int(math.log(n, k) + 0.5) == n def next_power_of_k(n, k): p = 1 while p < n: p *= k return p def ave(l): return sum(l) / len(l) def log_ceil(n, b): """ The smallest integer k such that b^k >= n. log_ceil(n, 2) is the number of bits needed to store any of n values, e.g. the number of bits needed to store any of 128 possible values is 7. """ p = 1 k = 0 while p < n: p *= b k += 1 return k def log_floor(n, b): """ The largest integer k such that b^k <= n. """ p = 1 k = 0 while p <= n: p *= b k += 1 return k - 1 def linear_fit_slope(ps): """ Single-independent-variable linear regression -- least squares method. At least, I *think* this function computes that answer. I no longer remember where I learned this trick and at the moment I can't prove to myself that this is correct. @param ps a sequence of tuples of (x, y) """ avex = ave([x for (x, y) in ps]) avey = ave([y for (x, y) in ps]) sxy = sum([ (x - avex) * (y - avey) for (x, y) in ps ]) sxx = sum([ (x - avex) ** 2 for (x, y) in ps ]) if sxx == 0: return None return sxy / sxx def permute(l): """ Return all possible permutations of l. @type l: sequence @rtype a set of sequences """ if len(l) == 1: return [l,] res = [] for i in range(len(l)): l2 = list(l[:]) x = l2.pop(i) for l3 in permute(l2): l3.append(x) res.append(l3) return res pyutil-3.3.2/pyutil/memutil.py000066400000000000000000000504241437014040400164230ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function # from the Python Standard Library import gc, math, operator, os, sys, types try: StandardError # PY2 except NameError: StandardError = Exception # PY3 # from the pyutil library from .assertutil import precondition from . import mathutil from . import dictutil class Canary: """ Want to get a printout when your object is garbage collected? Then put "self.canary = Canary(self)" in your object's constructor. """ def __init__(self, owner): self.ownerdesc = repr(owner) def __del__(self): print("Canary says that %s is gone." % self.ownerdesc) def estimate_mem_of_obj(o): # assumes 32-bit CPUs... PY_STRUCT_HEAD_LEN=4 if hasattr(o, '__len__'): if isinstance(o, str): return PY_STRUCT_HEAD_LEN + o.__len__() * 1 if isinstance(o, unicode): return PY_STRUCT_HEAD_LEN + o.__len__() * 4 # 4 depends on implementation and is approximate if isinstance(o, (tuple, list,)): return PY_STRUCT_HEAD_LEN + o.__len__() * 4 if isinstance(o, (dict, set,)): return PY_STRUCT_HEAD_LEN + o.__len__() * 4 * 2 * 2 # approximate if isinstance(o, int): return PY_STRUCT_HEAD_LEN + 4 if isinstance(o, long): return PY_STRUCT_HEAD_LEN + 4 if o < 1: return PY_STRUCT_HEAD_LEN else: return PY_STRUCT_HEAD_LEN + math.log(o) / 5 # the 5 was empirically determined (it is approximate) if isinstance(o, float): return PY_STRUCT_HEAD_LEN + 8 # Uh-oh... I wonder what we are missing here... return PY_STRUCT_HEAD_LEN def check_for_obj_leakage(f, *args, **kwargs): """ The idea is that I am going to invoke f(), then run gc.collect(), then run gc.get_objects() to get a complete list of all objects in the system, then invoke f() a second time, then run gc.collect(), then run gc.get_objects() to get a list of all the objects *now* in the system. Then I return a tuple two things: the first element of the tuple is the difference between the number of objects in the second list and the number of objects in the first list. I.e., if this number is zero then you can be pretty sure there is no memory leak, unless f is deleting some objects and replacing them by exactly the same number of objects but the new objects take up more memory. If this number is greater than zero then you can pretty sure there is a memory leak, unless f is doing some memoization/caching behavior and it will eventually stabilize, which you can detect by running check_for_obj_leakage() more times and seeing if it stabilizes. (Actually we run f() followed by gc.collect() one time before we start in order to account for any static objects which are created the first time you run f() and then re-used after that.) The second element in the return value is the set of all objects which were present in the second list and not in the first. Some of these objects might be memory-leaked objects, or perhaps f deleted some objects and replaced them with equivalent objects, in which case these objects are not leaked. (We actually invoke gc.collect() three times in a row in case there are objects which get collected in the first pass that have finalizers which create new reference-cycled objects... "3" is a superstitious number -- we figure most of the time the finalizers of the things produced by the first round of finalizers won't themselves product another round of reference-cycled objects.) """ f() gc.collect();gc.collect();gc.collect() f() gc.collect();gc.collect();gc.collect() r1 = gc.get_objects() f() gc.collect();gc.collect();gc.collect() r2 = gc.get_objects() d2 = dict([(id(x), x) for x in r2]) # Now remove everything from r1, and r1 itself, from d2. del d2[id(r1)] for o in r1: if id(o) in d2: del d2[id(o)] return (len(r2) - len(r1) - 1, d2) def measure_obj_leakage(f, numsamples=2**7, iterspersample=2**4, *args, **kwargs): """ The idea is we are going to use count_all_objects() to see how many objects are in use, and keep track of that number with respect to how many times we've invoked f(), and return the slope of the best linear fit. @param numsamples: recommended: 2**7 @param iterspersample: how many times f() should be invoked per sample; Basically, choose iterspersample such that iterspersample * numsamples * how-long-it-takes-to-compute-f() is slightly less than how long you are willing to wait for this leak test. @return: the slope of the best linear fit, which can be interpreted as 'the approximate number of Python objects created and not destroyed per invocation of f()' """ precondition(numsamples > 0, "numsamples is required to be positive.", numsamples) precondition(iterspersample > 0, "iterspersample is required to be positive.", iterspersample) resiters = [None]*numsamples # values: iters resnumobjs = [None]*numsamples # values: numobjs totaliters = 0 for i in range(numsamples): for j in range(iterspersample): f(*args, **kwargs) totaliters = totaliters + iterspersample resiters[i] = totaliters gc.collect() resnumobjs[i] = count_all_objects() # print "totaliters: %s, numobjs: %s" % (resiters[-1], resnumobjs[-1],) avex = float(reduce(operator.__add__, resiters)) / len(resiters) avey = float(reduce(operator.__add__, resnumobjs)) / len(resnumobjs) sxy = reduce(operator.__add__, map(lambda a, avex=avex, avey=avey: (a[0] - avex) * (a[1] - avey), zip(resiters, resnumobjs))) sxx = reduce(operator.__add__, map(lambda a, avex=avex: (a - avex) ** 2, resiters)) return sxy / sxx def linear_fit_slope(xs, ys): avex = float(reduce(operator.__add__, xs)) / len(xs) avey = float(reduce(operator.__add__, ys)) / len(ys) sxy = reduce(operator.__add__, map(lambda a, avex=avex, avey=avey: (a[0] - avex) * (a[1] - avey), zip(xs, ys))) sxx = reduce(operator.__add__, map(lambda a, avex=avex: (a - avex) ** 2, xs)) return sxy / sxx def measure_ref_leakage(f, numsamples=2**7, iterspersample=2**4, *args, **kwargs): """ The idea is we are going to use sys.gettotalrefcount() to see how many references are extant, and keep track of that number with respect to how many times we've invoked f(), and return the slope of the best linear fit. @param numsamples: recommended: 2**7 @param iterspersample: how many times f() should be invoked per sample; Basically, choose iterspersample such that iterspersample * numsamples * how-long-it-takes-to-compute-f() is slightly less than how long you are willing to wait for this leak test. @return: the slope of the best linear fit, which can be interpreted as 'the approximate number of Python references created and not nullified per invocation of f()' """ precondition(numsamples > 0, "numsamples is required to be positive.", numsamples) precondition(iterspersample > 0, "iterspersample is required to be positive.", iterspersample) try: sys.gettotalrefcount() except AttributeError as le: raise AttributeError(le, "Probably this is not a debug build of Python, so it doesn't have a sys.gettotalrefcount function.") resiters = [None]*numsamples # values: iters resnumrefs = [None]*numsamples # values: numrefs totaliters = 0 for i in range(numsamples): for j in range(iterspersample): f(*args, **kwargs) totaliters = totaliters + iterspersample resiters[i] = totaliters gc.collect() resnumrefs[i] = sys.gettotalrefcount() # print "totaliters: %s, numrefss: %s" % (resiters[-1], resnumrefs[-1],) avex = float(reduce(operator.__add__, resiters)) / len(resiters) avey = float(reduce(operator.__add__, resnumrefs)) / len(resnumrefs) sxy = reduce(operator.__add__, map(lambda a, avex=avex, avey=avey: (a[0] - avex) * (a[1] - avey), zip(resiters, resnumrefs))) sxx = reduce(operator.__add__, map(lambda a, avex=avex: (a - avex) ** 2, resiters)) return sxy / sxx class NotSupportedException(StandardError): """ Just an exception class. It is thrown by get_mem_usage if the OS does not support the operation. """ pass def get_mem_used(): """ This only works on Linux, and only if the /proc/$PID/statm output is the same as that in linux kernel 2.6. Also `os.getpid()' must work. @return: tuple of (res, virt) used by this process """ try: import resource except ImportError: raise NotSupportedException() # sample output from cat /proc/$PID/statm: # 14317 3092 832 279 0 2108 0 a = os.popen("cat /proc/%s/statm 2>/dev/null" % os.getpid()).read().split() #nosec if not a: raise NotSupportedException() return (int(a[1]) * resource.getpagesize(), int(a[0]) * resource.getpagesize(),) def get_mem_used_res(): """ This only works on Linux, and only if the /proc/$PID/statm output is the same as that in linux kernel 2.6. Also `os.getpid()' must work. """ try: import resource except ImportError: raise NotSupportedException() # sample output from cat /proc/$PID/statm: # 14317 3092 832 279 0 2108 0 a = os.popen("cat /proc/%s/statm" % os.getpid()).read().split() #nosec if not len(a) > 1: raise NotSupportedException() return int(a[1]) * resource.getpagesize() def get_mem_usage_virt_and_res(): """ This only works on Linux, and only if the /proc/$PID/statm output is the same as that in linux kernel 2.6. Also `os.getpid()' must work. """ try: import resource except ImportError: raise NotSupportedException() # sample output from cat /proc/$PID/statm: # 14317 3092 832 279 0 2108 0 a = os.popen("cat /proc/%s/statm" % os.getpid()).read().split() #nosec if not len(a) > 1: raise NotSupportedException() return (int(a[0]) * resource.getpagesize(), int(a[1]) * resource.getpagesize(),) class Measurer(object): def __init__(self, f, numsamples=2**7, iterspersample=2**4, *args, **kwargs): """ @param f a callable; If it returns a deferred then the memory will not be measured and the next iteration will not be started until the deferred fires; else the memory will be measured and the next iteration started when f returns. """ self.f = f self.numsamples = numsamples self.iterspersample = iterspersample self.args = args self.kwargs = kwargs # from twisted from twisted.internet import defer self.d = defer.Deferred() def when_complete(self): return self.d def _invoke(self): d = self.f(*self.args, **self.kwargs) # from twisted from twisted.internet import defer if isinstance(d, defer.Deferred): d.addCallback(self._after) else: self._after(None) def start(self): self.resiters = [None]*self.numsamples # values: iters self.resmemusage = [None]*self.numsamples # values: memusage self.totaliters = 0 self.i = 0 self.j = 0 self._invoke() def _after(self, o): self.j += 1 if self.j < self.iterspersample: self._invoke() return if self.i < self.numsamples: self.j = 0 self.i += 1 self.totaliters += self.iterspersample self.resiters[self.i] = self.totaliters self.resmemusage[self.i] = get_mem_used_res() self._invoke() return self.d.callback(mathutil.linear_fit_slope(zip(self.resiters, self.resmemusage))) def measure_mem_leakage(f, numsamples=2**7, iterspersample=2**4, *args, **kwargs): """ This does the same thing as measure_obj_leakage() but instead of using count_all_objects() it uses get_mem_usage(), which is currently implemented for Linux and barely implemented for Mac OS X. @param numsamples: recommended: 2**7 @param iterspersample: how many times `f()' should be invoked per sample; Basically, choose `iterspersample' such that (iterspersample * numsamples * how-long-it-takes-to-compute-`f()') is slightly less than how long you are willing to wait for this leak test. @return: the slope of the best linear fit, which can be interpreted as 'the approximate number of system bytes allocated and not freed per invocation of f()' """ precondition(numsamples > 0, "numsamples is required to be positive.", numsamples) precondition(iterspersample > 0, "iterspersample is required to be positive.", iterspersample) resiters = [None]*numsamples # values: iters resmemusage = [None]*numsamples # values: memusage totaliters = 0 for i in range(numsamples): for j in range(iterspersample): f(*args, **kwargs) totaliters = totaliters + iterspersample resiters[i] = totaliters gc.collect() resmemusage[i] = get_mem_used_res() # print("totaliters: %s, numobjs: %s" % (resiters[-1], resmemusage[-1],)) avex = float(reduce(operator.__add__, resiters)) / len(resiters) avey = float(reduce(operator.__add__, resmemusage)) / len(resmemusage) sxy = reduce(operator.__add__, map(lambda a, avex=avex, avey=avey: (a[0] - avex) * (a[1] - avey), zip(resiters, resmemusage))) sxx = reduce(operator.__add__, map(lambda a, avex=avex: (a - avex) ** 2, resiters)) if sxx == 0: return None return sxy / sxx def describe_object(o, FunctionType=types.FunctionType, MethodType=types.MethodType): """ For human analysis, when humans are attempting to understand where all the memory is going. Argument o is an object, return value is a string describing the object. """ sl = [] if isinstance(o, FunctionType): try: sl.append("" % str(o.func_name)) except: pass elif isinstance(o, MethodType): try: sl.append("" % str(o.im_func.func_name)) except: pass else: sl.append(str(type(o))) try: sl.append(str(len(o))) except: pass return ''.join(sl) def describe_object_with_dict_details(o): sl = [] sl.append(str(type(o))) if isinstance(o, types.FunctionType): try: sl.append(str(o.func_name)) except: pass elif isinstance(o, types.MethodType): try: sl.append(str(o.im_func.func_name)) except: pass try: sl.append(str(len(o))) except: pass if isinstance(o, dict) and o: sl.append('-') nd = dictutil.NumDict() for k, v in o.iteritems(): nd.inc((describe_object(k), describe_object(v),)) k, v = nd.item_with_largest_value() sl.append("-") iterator = o.iteritems() k,v = iterator.next() sl.append(describe_object(k)) sl.append(":") sl.append(describe_object(v)) return ''.join(sl) def describe_dict(o): sl = ['') return ''.join(sl) def count_all_objects(): ids = set() ls = locals() import inspect cf = inspect.currentframe() for o in gc.get_objects(): if o is ids or o is ls or o is cf: continue if not id(o) in ids: ids.add(id(o)) for so in gc.get_referents(o): if not id(so) in ids: ids.add(id(so)) return len(ids) def visit_all_objects(f): """ Brian and I *think* that this gets all objects. This is predicated on the assumption that every object either participates in gc, or is at most one hop from an object that participates in gc. This was Brian's clever idea. """ ids = set() ls = locals() import inspect cf = inspect.currentframe() for o in gc.get_objects(): if o is ids or o is ls or o is cf: continue if not id(o) in ids: ids.add(id(o)) f(o) for so in gc.get_referents(o): if not id(so) in ids: ids.add(id(so)) f(so) def get_all_objects(): objs = [] def addit(o): objs.append(o) visit_all_objects(addit) return objs def describe_all_objects(): import dictutil d = dictutil.NumDict() for o in get_all_objects(): d.inc(describe_object(o)) return d def dump_description_of_object(o, f): f.write("%x" % (id(o),)) f.write("-") f.write(describe_object(o)) f.write("\n") def dump_description_of_object_refs(o, f): # This holds the ids of all referents that we've already dumped. dumped = set() # First, any __dict__ items try: itemsiter = o.__dict__.iteritems() except: pass else: for k, v in itemsiter: try: idr = id(v) if idr not in dumped: dumped.add(idr) f.write("%d:"%len(k)) f.write(k) f.write(",") f.write("%0x,"%idr) except: pass # Then anything else that gc.get_referents() returns. for r in gc.get_referents(o): idr = id(r) if idr not in dumped: dumped.add(idr) f.write("0:,%0x,"%idr) def dump_descriptions_of_all_objects(f): ids = set() ls = locals() for o in gc.get_objects(): if o is f or o is ids or o is ls: continue if not id(o) in ids: ids.add(id(o)) dump_description_of_object(o, f) for so in gc.get_referents(o): if o is f or o is ids or o is ls: continue if not id(so) in ids: ids.add(id(so)) dump_description_of_object(so, f) ls = None # break reference cycle return len(ids) def dump_description_of_object_with_refs(o, f): f.write("%0x" % (id(o),)) f.write("-") desc = describe_object(o) f.write("%d:"%len(desc)) f.write(desc) f.write(",") dump_description_of_object_refs(o, f) f.write("\n") def dump_descriptions_of_all_objects_with_refs(f): ids = set() ls = locals() for o in gc.get_objects(): if o is f or o is ids or o is ls: continue if not id(o) in ids: ids.add(id(o)) dump_description_of_object_with_refs(o, f) for so in gc.get_referents(o): if o is f or o is ids or o is ls: continue if not id(so) in ids: ids.add(id(so)) dump_description_of_object_with_refs(so, f) ls = None # break reference cycle return len(ids) import re NRE = re.compile("[1-9][0-9]*$") def undump_descriptions_of_all_objects(inf): d = {} for l in inf: dash=l.find('-') if dash == -1: raise l mo = NRE.search(l) if mo: typstr = l[dash+1:mo.start(0)] num=int(mo.group(0)) if str(num) != mo.group(0): raise mo.group(0) else: typstr = l[dash+1:] num = None d[l[:dash]] = (typstr, num,) return d pyutil-3.3.2/pyutil/nummedobj.py000066400000000000000000000041461437014040400167270ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from . import dictutil class NummedObj(object): """ This is useful for nicer debug printouts. Instead of objects of the same class being distinguished from one another by their memory address, they each get a unique number, which can be read as "the first object of this class", "the second object of this class", etc. This is especially useful because separate runs of a program will yield identical debug output, (assuming that the objects get created in the same order in each run). This makes it possible to diff outputs from separate runs to see what changed, without having to ignore a difference on every line due to different memory addresses of objects. """ objnums = dictutil.NumDict() # key: class names, value: highest used object number def __init__(self, klass=None): """ @param klass: in which class are you counted? If default value of `None', then self.__class__ will be used. """ if klass is None: klass = self.__class__ self._classname = klass.__name__ NummedObj.objnums.inc(self._classname) self._objid = NummedObj.objnums[self._classname] def __repr__(self): return "<%s #%d>" % (self._classname, self._objid,) def __lt__(self, other): return (self._objid, self._classname,) < (other._objid, other._classname,) def __le__(self, other): return (self._objid, self._classname,) <= (other._objid, other._classname,) def __eq__(self, other): return (self._objid, self._classname,) == (other._objid, other._classname,) def __ne__(self, other): return (self._objid, self._classname,) != (other._objid, other._classname,) def __gt__(self, other): return (self._objid, self._classname,) > (other._objid, other._classname,) def __ge__(self, other): return (self._objid, self._classname,) >= (other._objid, other._classname,) def __hash__(self): return id(self) pyutil-3.3.2/pyutil/observer.py000066400000000000000000000061111437014040400165700ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # -*- test-case-name: allmydata.test.test_observer -*- from twisted.internet import defer try: from foolscap.eventual import eventually eventually # http://divmod.org/trac/ticket/1499 except ImportError: from twisted.internet import reactor def eventually(f, *args, **kwargs): return reactor.callLater(0, f, *args, **kwargs) """The idiom we use is for the observed object to offer a method named 'when_something', which returns a deferred. That deferred will be fired when something happens. The way this is typically implemented is that the observed has an ObserverList whose when_fired method is called in the observed's 'when_something'.""" class OneShotObserverList: """A one-shot event distributor.""" def __init__(self): self._fired = False self._result = None self._watchers = [] def __repr__(self): if self._fired: return " %s>" % (self._result, ) else: return "" % (self._watchers, ) def _get_result(self): return self._result def when_fired(self): if self._fired: return defer.succeed(self._get_result()) d = defer.Deferred() self._watchers.append(d) return d def fire(self, result): assert not self._fired self._fired = True self._result = result self._fire(result) def _fire(self, result): for w in self._watchers: eventually(w.callback, result) del self._watchers def fire_if_not_fired(self, result): if not self._fired: self.fire(result) class LazyOneShotObserverList(OneShotObserverList): """ a variant of OneShotObserverList which does not retain the result it handles, but rather retains a callable() through which is retrieves the data if and when needed. """ def __init__(self): OneShotObserverList.__init__(self) def _get_result(self): return self._result_producer() def fire(self, result_producer): """ @param result_producer: a no-arg callable which returns the data which is to be considered the 'result' for this observer list. note that this function may be called multiple times - once upon initial firing, and potentially once more for each subsequent when_fired() deferred created """ assert not self._fired self._fired = True self._result_producer = result_producer if self._watchers: # if not, don't call result_producer self._fire(self._get_result()) class ObserverList: """A simple class to distribute events to a number of subscribers.""" def __init__(self): self._watchers = [] def subscribe(self, observer): self._watchers.append(observer) def unsubscribe(self, observer): self._watchers.remove(observer) def notify(self, *args, **kwargs): for o in self._watchers: eventually(o, *args, **kwargs) pyutil-3.3.2/pyutil/odict.py000066400000000000000000000525771437014040400160640ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ This module offers a Ordered Dict, which is a dict that preserves insertion order. See PEP 372 for description of the problem. This implementation uses a linked-list to get good O(1) asymptotic performance. (Actually it is O(hashtable-update-cost), but whatever.) Warning: if -O optimizations are not turned on then OrderedDict performs extensive self-analysis in every function call, which can take minutes and minutes for a large cache. Turn on -O, or comment out assert self._assert_invariants() """ import operator from .assertutil import _assert, precondition from .humanreadable import hr class OrderedDict: """ An efficient ordered dict. Adding an item that is already in the dict *does not* make it the most- recently-added item although it may change the state of the dict itself (if the value is different than the previous value). See also SmallOrderedDict (below), which is faster in some cases. """ class ItemIterator: def __init__(self, c): self.c = c self.i = c.d[c.ts][1] def __iter__(self): return self def __next__(self): if self.i is self.c.hs: raise StopIteration() k = self.i precondition(k in self.c.d, "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) (v, p, n,) = self.c.d[k] self.i = p return (k, v,) def next(self): return self.__next__() class KeyIterator: def __init__(self, c): self.c = c self.i = c.d[c.ts][1] def __iter__(self): return self def __next__(self): if self.i is self.c.hs: raise StopIteration() k = self.i precondition(k in self.c.d, "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c) (v, p, n,) = self.c.d[k] self.i = p return k def next(self): return self.__next__() class ValIterator: def __init__(self, c): self.c = c self.i = c.d[c.ts][1] def __iter__(self): return self def __next__(self): if self.i is self.c.hs: raise StopIteration() precondition(self.i in self.c.d, "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) (v, p, n,) = self.c.d[self.i] self.i = p return v def next(self): return self.__next__() class Sentinel: def __init__(self, msg): self.msg = msg def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.msg,) def __init__(self, initialdata={}): self.d = {} # k: k, v: [v, prev, next,] # the dict self.hs = OrderedDict.Sentinel("hs") self.ts = OrderedDict.Sentinel("ts") self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. self.update(initialdata) assert self._assert_invariants() def __repr_n__(self, n=None): s = ["{",] try: iter = self.iteritems() x = iter.next() s.append(str(x[0])); s.append(": "); s.append(str(x[1])) i = 1 while (n is None) or (i < n): x = iter.next() s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1])) except StopIteration: pass s.append("}") return ''.join(s) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),) def __str__(self): return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),) def _assert_invariants(self): _assert((len(self.d) > 2) == (self.d[self.hs][2] is not self.ts) == (self.d[self.ts][1] is not self.hs), "Head and tail point to something other than each other if and only if there is at least one element in the dictionary.", self.hs, self.ts, len(self.d)) foundprevsentinel = 0 foundnextsentinel = 0 for (k, (v, p, n,)) in self.d.items(): _assert(v not in (self.hs, self.ts,)) _assert(p is not self.ts, "A reference to the tail sentinel may not appear in prev.", k, v, p, n) _assert(n is not self.hs, "A reference to the head sentinel may not appear in next.", k, v, p, n) _assert(p in self.d, "Each prev is required to appear as a key in the dict.", k, v, p, n) _assert(n in self.d, "Each next is required to appear as a key in the dict.", k, v, p, n) if p is self.hs: foundprevsentinel += 1 _assert(foundprevsentinel <= 2, "No more than two references to the head sentinel may appear as a prev.", k, v, p, n) if n is self.ts: foundnextsentinel += 1 _assert(foundnextsentinel <= 2, "No more than one reference to the tail sentinel may appear as a next.", k, v, p, n) _assert(foundprevsentinel == 2, "A reference to the head sentinel is required appear as a prev (plus a self-referential reference).") _assert(foundnextsentinel == 2, "A reference to the tail sentinel is required appear as a next (plus a self-referential reference).") count = 0 for (k, v,) in self.iteritems(): _assert(k not in (self.hs, self.ts,), k, self.hs, self.ts) count += 1 _assert(count == len(self.d)-2, count, len(self.d)) # -2 for the sentinels return True def move_to_most_recent(self, k, strictkey=False): assert self._assert_invariants() if k not in self.d: if strictkey: raise KeyError(k) return node = self.d[k] # relink self.d[node[1]][2] = node[2] self.d[node[2]][1] = node[1] # move to front hnode = self.d[self.hs] node[1] = self.hs node[2] = hnode[2] hnode[2] = k self.d[node[2]][1] = k assert self._assert_invariants() def iteritems(self): return OrderedDict.ItemIterator(self) def itervalues(self): return OrderedDict.ValIterator(self) def iterkeys(self): return self.__iter__() def __iter__(self): return OrderedDict.KeyIterator(self) def __getitem__(self, key, default=None, strictkey=True): node = self.d.get(key) if not node: if strictkey: raise KeyError(key) return default return node[0] def __setitem__(self, k, v=None): assert self._assert_invariants() node = self.d.get(k) if node: node[0] = v return hnode = self.d[self.hs] n = hnode[2] self.d[k] = [v, self.hs, n,] hnode[2] = k self.d[n][1] = k assert self._assert_invariants() return v def __delitem__(self, key, default=None, strictkey=True): """ @param strictkey: True if you want a KeyError in the case that key is not there, False if you want a reference to default in the case that key is not there @param default: the object to return if key is not there; This is ignored if strictkey. @return: the value removed or default if there is not item by that key and strictkey is False """ assert self._assert_invariants() if key in self.d: node = self.d[key] # relink self.d[node[1]][2] = node[2] self.d[node[2]][1] = node[1] del self.d[key] assert self._assert_invariants() return node[0] elif strictkey: assert self._assert_invariants() raise KeyError(key) else: assert self._assert_invariants() return default def has_key(self, key): assert self._assert_invariants() if key in self.d: assert self._assert_invariants() return True else: assert self._assert_invariants() return False def __contains__(self, key): return self.has_key(key) def clear(self): assert self._assert_invariants() self.d.clear() self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes. assert self._assert_invariants() def update(self, otherdict): """ @return: self """ assert self._assert_invariants() for (k, v,) in otherdict.items(): assert self._assert_invariants() self[k] = v assert self._assert_invariants() def pop(self): assert self._assert_invariants() if len(self.d) < 2: # the +2 is for the sentinels raise KeyError('popitem(): dictionary is empty') k = self.d[self.hs][2] self.remove(k) assert self._assert_invariants() return k def popitem(self): assert self._assert_invariants() if len(self.d) < 2: # the +2 is for the sentinels raise KeyError('popitem(): dictionary is empty') k = self.d[self.hs][2] val = self.remove(k) assert self._assert_invariants() return (k, val,) def keys_unsorted(self): assert self._assert_invariants() t = self.d.copy() del t[self.hs] del t[self.ts] assert self._assert_invariants() return t.keys() if hasattr(dict, 'iterkeys'): # PY2 def keys(self): res = [None] * len(self) i = 0 for k in self.iterkeys(): res[i] = k i += 1 return res else: # PY3 def keys(self): return self.__iter__() def values_unsorted(self): assert self._assert_invariants() t = self.d.copy() del t[self.hs] del t[self.ts] assert self._assert_invariants() return map(operator.__getitem__, t.values(), [0]*len(t)) if hasattr(dict, 'itervalues'): # PY2 def values(self): res = [None] * len(self) i = 0 for v in self.itervalues(): res[i] = v i += 1 return res else: # PY3 def values(self): return OrderedDict.ValIterator(self) if hasattr(dict, 'iteritems'): # PY2 def items(self): res = [None] * len(self) i = 0 for it in self.iteritems(): res[i] = it i += 1 return res else: # PY3 def items(self): return OrderedDict.ItemIterator(self) def __len__(self): return len(self.d) - 2 def insert(self, key, val=None): assert self._assert_invariants() result = self.__setitem__(key, val) assert self._assert_invariants() return result def setdefault(self, key, default=None): assert self._assert_invariants() if not self.has_key(key): self[key] = default assert self._assert_invariants() return self[key] def get(self, key, default=None): return self.__getitem__(key, default, strictkey=False) def remove(self, key, default=None, strictkey=True): assert self._assert_invariants() result = self.__delitem__(key, default, strictkey) assert self._assert_invariants() return result class SmallOrderedDict(dict): """ SmallOrderedDict is faster than OrderedDict for small sets. How small? That depends on your machine and which operations you use most often. Use performance profiling to determine whether the ordered dict class that you are using makes any difference to the performance of your program, and if it does, then run "quick_bench()" in test/test_cache.py to see which cache implementation is faster for the size of your datasets. A simple least-recently-used cache. It keeps an LRU queue, and when the number of items in the cache reaches maxsize, it removes the least recently used item. "Looking" at an item or a key such as with "has_key()" makes that item become the most recently used item. You can also use "refresh()" to explicitly make an item become the most recently used item. Adding an item that is already in the dict *does* make it the most- recently-used item although it does not change the state of the dict itself. """ class ItemIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def next(self): precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) precondition(dict.__contains__(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) if self.i == len(self.c._lru): raise StopIteration() k = self.i self.i += 1 return (k, dict.__getitem__(self.c, k),) class KeyIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def next(self): precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) precondition(dict.__contains__(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) if self.i == len(self.c._lru): raise StopIteration() k = self.i self.i += 1 return k class ValueIterator: def __init__(self, c): self.c = c self.i = 0 def __iter__(self): return self def next(self): precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c) precondition(dict.__contains__(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c) if self.i == len(self.c._lru): raise StopIteration() k = self.i self.i += 1 return dict.__getitem__(self.c, k) def __init__(self, initialdata={}, maxsize=128): dict.__init__(self, initialdata) self._lru = initialdata.keys() # contains keys self._maxsize = maxsize over = len(self) - self._maxsize if over > 0: map(dict.__delitem__, [self]*over, self._lru[:over]) del self._lru[:over] assert self._assert_invariants() def _assert_invariants(self): _assert(len(self._lru) <= self._maxsize, "Size is required to be <= maxsize.") _assert(len(filter(lambda x: dict.__contains__(self, x), self._lru)) == len(self._lru), "Each key in self._lru is required to be in dict.", filter(lambda x: dict.__contains__(self, x), self._lru), len(self._lru), self._lru, len(self), self) _assert(len(filter(lambda x: x in self._lru, self.keys())) == len(self), "Each key in dict is required to be in self._lru.", filter(lambda x: x not in self._lru, self.keys()), len(self._lru), self._lru, len(self), self) _assert(len(self._lru) == len(self), "internal consistency", filter(lambda x: x not in self.keys(), self._lru), len(self._lru), self._lru, len(self), self) _assert(len(self._lru) <= self._maxsize, "internal consistency", len(self._lru), self._lru, self._maxsize) return True def insert(self, key, item=None): assert self._assert_invariants() result = self.__setitem__(key, item) assert self._assert_invariants() return result def setdefault(self, key, default=None): assert self._assert_invariants() if dict.__contains__(self, key): self[key] = default assert self._assert_invariants() return self[key] def __setitem__(self, key, item=None): assert self._assert_invariants() if dict.__contains__(self, key): self._lru.remove(key) else: if len(self._lru) == self._maxsize: # If this insert is going to increase the size of the cache to bigger than maxsize: killkey = self._lru.pop(0) dict.__delitem__(self, killkey) dict.__setitem__(self, key, item) self._lru.append(key) assert self._assert_invariants() return item def remove(self, key, default=None, strictkey=True): assert self._assert_invariants() result = self.__delitem__(key, default, strictkey) assert self._assert_invariants() return result def __delitem__(self, key, default=None, strictkey=True): """ @param strictkey: True if you want a KeyError in the case that key is not there, False if you want a reference to default in the case that key is not there @param default: the object to return if key is not there; This is ignored if strictkey. @return: the object removed or default if there is not item by that key and strictkey is False """ assert self._assert_invariants() if dict.__contains__(self, key): val = dict.__getitem__(self, key) dict.__delitem__(self, key) self._lru.remove(key) assert self._assert_invariants() return val elif strictkey: assert self._assert_invariants() raise KeyError(key) else: assert self._assert_invariants() return default def clear(self): assert self._assert_invariants() dict.clear(self) self._lru = [] assert self._assert_invariants() def update(self, otherdict): """ @return: self """ assert self._assert_invariants() if len(otherdict) > self._maxsize: # Handling this special case here makes it possible to implement the # other more common cases faster below. dict.clear(self) self._lru = [] if self._maxsize > (len(otherdict) - self._maxsize): dict.update(self, otherdict) while len(self) > self._maxsize: dict.popitem(self) else: for k, v, in otherdict.iteritems(): if len(self) == self._maxsize: break dict.__setitem__(self, k, v) self._lru = dict.keys(self) assert self._assert_invariants() return self for k in otherdict.iterkeys(): if dict.__contains__(self, k): self._lru.remove(k) self._lru.extend(otherdict.keys()) dict.update(self, otherdict) over = len(self) - self._maxsize if over > 0: map(dict.__delitem__, [self]*over, self._lru[:over]) del self._lru[:over] assert self._assert_invariants() return self def has_key(self, key): assert self._assert_invariants() if dict.__contains__(self, key): assert key in self._lru, "key: %s, self._lru: %s" % tuple(map(hr, (key, self._lru,))) self._lru.remove(key) self._lru.append(key) assert self._assert_invariants() return True else: assert self._assert_invariants() return False def __contains__(self, key): return self.has_key(key) def refresh(self, key, strictkey=True): """ @param strictkey: raise a KeyError exception if key isn't present """ assert self._assert_invariants() if not dict.__contains__(self, key): if strictkey: raise KeyError(key) return self._lru.remove(key) self._lru.append(key) def popitem(self): if not self._lru: raise KeyError('popitem(): dictionary is empty') k = self._lru[-1] obj = self.remove(k) return (k, obj,) pyutil-3.3.2/pyutil/platformutil.py000066400000000000000000000071401437014040400174660ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # Thanks to Daenyth for help porting this to Arch Linux. import os, platform, re, subprocess _distributor_id_cmdline_re = re.compile("(?:Distributor ID:)\s*(.*)", re.I) _release_cmdline_re = re.compile("(?:Release:)\s*(.*)", re.I) _distributor_id_file_re = re.compile("(?:DISTRIB_ID\s*=)\s*(.*)", re.I) _release_file_re = re.compile("(?:DISTRIB_RELEASE\s*=)\s*(.*)", re.I) global _distname,_version _distname = None _version = None def get_linux_distro(): """ Tries to determine the name of the Linux OS distribution name. First, try to parse a file named "/etc/lsb-release". If it exists, and contains the "DISTRIB_ID=" line and the "DISTRIB_RELEASE=" line, then return the strings parsed from that file. If that doesn't work, then invoke platform.dist(). If that doesn't work, then try to execute "lsb_release", as standardized in 2001: http://refspecs.freestandards.org/LSB_1.0.0/gLSB/lsbrelease.html The current version of the standard is here: http://refspecs.freestandards.org/LSB_3.2.0/LSB-Core-generic/LSB-Core-generic/lsbrelease.html that lsb_release emitted, as strings. Returns a tuple (distname,version). Distname is what LSB calls a "distributor id", e.g. "Ubuntu". Version is what LSB calls a "release", e.g. "8.04". A version of this has been submitted to python as a patch for the standard library module "platform": http://bugs.python.org/issue3937 """ global _distname,_version if _distname and _version: return (_distname, _version) try: etclsbrel = open("/etc/lsb-release", "rU") for line in etclsbrel: m = _distributor_id_file_re.search(line) if m: _distname = m.group(1).strip() if _distname and _version: return (_distname, _version) m = _release_file_re.search(line) if m: _version = m.group(1).strip() if _distname and _version: return (_distname, _version) except EnvironmentError: pass (_distname, _version) = platform.dist()[:2] if _distname and _version: return (_distname, _version) try: p = subprocess.Popen(["lsb_release", "--all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) rc = p.wait() if rc == 0: for line in p.stdout.readlines(): m = _distributor_id_cmdline_re.search(line) if m: _distname = m.group(1).strip() if _distname and _version: return (_distname, _version) m = _release_cmdline_re.search(p.stdout.read()) if m: _version = m.group(1).strip() if _distname and _version: return (_distname, _version) except EnvironmentError: pass if os.path.exists("/etc/arch-release"): return ("Arch_Linux", "") return (_distname,_version) def get_platform(): # Our version of platform.platform(), telling us both less and more than the # Python Standard Library's version does. # We omit details such as the Linux kernel version number, but we add a # more detailed and correct rendition of the Linux distribution and # distribution-version. if "linux" in platform.system().lower(): return platform.system()+"-"+"_".join(get_linux_distro())+"-"+platform.machine()+"-"+"_".join([x for x in platform.architecture() if x]) else: return platform.platform() pyutil-3.3.2/pyutil/randutil.py000066400000000000000000000025771437014040400165770ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import warnings import os, random class devrandomRandom(random.Random): """ The problem with using this one, of course, is that it blocks. This is, of course, a security flaw. (On Linux and probably on other systems.) --Zooko 2005-03-04 Not repeatable. """ def __init__(self): warnings.warn("deprecated", DeprecationWarning) self.dr = open("/dev/random", "r") def get(self, bytes): return self.dr.read(bytes) class devurandomRandom(random.Random): """ The problem with using this one is that it gives answers even when it has never been properly seeded, e.g. when you are booting from CD and have just started up and haven't yet gathered enough entropy to actually be unguessable. (On Linux and probably on other systems.) --Zooko 2005-03-04 Not repeatable. """ def get(self, bytes): warnings.warn("deprecated", DeprecationWarning) return os.urandom(bytes) randobj = devurandomRandom() get = randobj.get random = randobj.random randrange = randobj.randrange shuffle = randobj.shuffle choice = randobj.choice seed = randobj.seed def randstr(n): return ''.join(map(chr, map(randrange, [0]*n, [256]*n))) def insecurerandstr(n): return os.urandom(n) pyutil-3.3.2/pyutil/repeatable_random.py000066400000000000000000000071571437014040400204200ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- """ If you execute force_repeatability() then the following things are changed in the runtime: 1. random.random() and its sibling functions, and random.Random.seed() in the random module are seeded with a known seed so that they will return the same sequence on each run. 2. os.urandom() is replaced by a fake urandom that returns a pseudorandom sequence. 3. time.time() is replaced by a fake time that returns an incrementing number. (Original time.time is available as time.realtime.) Which seed will be used? If the environment variable REPEATABLE_RANDOMNESS_SEED is set, then it will use that. Else, it will use the current real time. In either case it logs the seed that it used. Caveats: 1. If some code has acquired a random.Random object before force_repeatability() is executed, then that Random object will produce non-reproducible results. For example, the tempfile module in the Python Standard Library does this. 2. Likewise if some code called time.time() before force_repeatability() was called, then it will have gotten a real time stamp. For example, trial does this. (Then it later subtracts that real timestamp from a faketime timestamp to calculate elapsed time, resulting in a large negative elapsed time.) 3. Fake urandom has an added constraint for performance reasons -- you can't ask it for more than 64 bytes of randomness at a time. (I couldn't figure out how to generate large fake random strings efficiently.) """ import os, random, time if not hasattr(time, "realtime"): time.realtime = time.time if not hasattr(os, "realurandom"): os.realurandom = os.urandom if not hasattr(random, "realseed"): random.realseed = random.seed tdelta = 0 seeded = False def force_repeatability(): now = 1043659734.0 def faketime(): global tdelta tdelta += 1 return now + tdelta time.faketime = faketime time.time = faketime from idlib import i2b def fakeurandom(n): if n > 64: raise ("Can't produce more than 64 bytes of pseudorandomness efficiently.") elif n == 0: return '' else: z = i2b(random.getrandbits(n*8)) x = z + "0" * (n-len(z)) assert len(x) == n return x os.fakeurandom = fakeurandom os.urandom = fakeurandom global seeded if not seeded: SEED = os.environ.get('REPEATABLE_RANDOMNESS_SEED', None) if SEED is None: # Generate a seed which is integral and fairly short (to ease cut-and-paste, writing it down, etc.). t = time.realtime() subsec = t % 1 t += (subsec * 1000000) t %= 1000000 SEED = long(t) import sys sys.stdout.write("REPEATABLE_RANDOMNESS_SEED: %s\n" % SEED) ; sys.stdout.flush() sys.stdout.write("In order to reproduce this run of the code, set the environment variable \"REPEATABLE_RANDOMNESS_SEED\" to %s before executing.\n" % SEED) ; sys.stdout.flush() random.seed(SEED) def seed_which_refuses(a): sys.stdout.write("I refuse to reseed to %s. Go away!\n" % (a,)) ; sys.stdout.flush() return random.realseed = random.seed random.seed = seed_which_refuses seeded = True import setutil setutil.RandomSet.DETERMINISTIC = True def restore_real_clock(): time.time = time.realtime def restore_real_urandom(): os.urandom = os.realurandom def restore_real_seed(): random.seed = random.realseed def restore_non_repeatability(): restore_real_seed() restore_real_urandom() restore_real_clock() pyutil-3.3.2/pyutil/scripts/000077500000000000000000000000001437014040400160575ustar00rootroot00000000000000pyutil-3.3.2/pyutil/scripts/__init__.py000077500000000000000000000000001437014040400201610ustar00rootroot00000000000000pyutil-3.3.2/pyutil/scripts/lines.py000077500000000000000000000014261437014040400175510ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from pyutil import lineutil import sys def main(): if len(sys.argv) > 1 and "-s" in sys.argv[1:]: strip = True sys.argv.remove("-s") else: strip = False if len(sys.argv) > 1 and "-n" in sys.argv[1:]: nobak = True sys.argv.remove("-n") else: nobak = False if len(sys.argv) > 1: pipe = False else: pipe = True if pipe: lineutil.lineify_fileobjs(sys.stdin, sys.stdout) else: for fn in sys.argv[1:]: lineutil.lineify_file(fn, strip, nobak) if __name__ == '__main__': main() pyutil-3.3.2/pyutil/scripts/memdump2dot.py000077500000000000000000000033311437014040400206710ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import bindann bindann.install_exception_handler() import sys inf = open(sys.argv[1], "r") outf = open(sys.argv[1]+".dot", "w") outf.write("digraph %s {\n" % sys.argv[1].replace(".","")) def parse_netstring(l, i): try: j = l.find(':', i) if j == -1: return (None, len(l),) lenval = int(l[i:j]) val = l[j+1:j+1+lenval] # skip the comma assert l[j+1+lenval] == "," return (val, j+1+lenval+1,) except Exception as le: le.args = tuple(le.args + (l, i,)) raise def parse_ref(l, i): (attrname, i,) = parse_netstring(l, i) j = l.find(",", i) assert j != -1 objid = l[i:j] return (objid, attrname, j+1,) def parse_memdump_line(l): result = [] i = l.find('-') objid = l[:i] (objdesc, i,) = parse_netstring(l, i+1) result.append((objid, objdesc,)) while i != -1 and i < len(l): (objid, attrname, i,) = parse_ref(l, i) result.append((objid, attrname,)) return result for l in inf: if l[-1] != "\n": raise "waht the HECK? %r" % l res = parse_memdump_line(l.strip()) # declare the node outf.write("\"%s\" [label=\"%s\"];\n" % (res[0][0], res[0][1],)) # declare all the edges for edge in res[1:]: if edge[1]: # a named edge outf.write("\"%s\" -> \"%s\" [style=bold, label=\"%s\"];\n" % (res[0][0], edge[0], edge[1],)) else: # an anonymous edge outf.write("\"%s\" -> \"%s\";\n" % (res[0][0], edge[0])) outf.write("}") pyutil-3.3.2/pyutil/scripts/passphrase.py000066400000000000000000000056151437014040400206110ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import argparse, math, random, sys from pyutil.mathutil import div_ceil from pkg_resources import resource_stream def recursive_subset_sum(entropy_needed, wordlists): # Pick a minimalish set of numbers which sum to at least # entropy_needed. # Okay now what's the smallest number of words which will give us # at least this much entropy? entropy_of_biggest_wordlist = wordlists[-1][0] assert isinstance(entropy_of_biggest_wordlist, float), wordlists[-1] needed_words = div_ceil(entropy_needed, entropy_of_biggest_wordlist) # How much entropy do we need from each word? needed_entropy_per_word = entropy_needed / needed_words # What's the smallest wordlist that offers at least this much # entropy per word? for (wlentropy, wl) in wordlists: if wlentropy >= needed_entropy_per_word: break assert wlentropy >= needed_entropy_per_word, (wlentropy, needed_entropy_per_word) result = [(wlentropy, wl)] # If we need more, recurse... if wlentropy < entropy_needed: rest = recursive_subset_sum(entropy_needed - wlentropy, wordlists) result.extend(rest) return result def gen_passphrase(entropy, allwords): maxlenwords = [] i = 2 # The smallest set is words of length 1 or 2. words = [x for x in allwords if len(x) <= i] maxlenwords.append((math.log(len(words), 2), words)) while len(maxlenwords[-1][1]) < len(allwords): i += 1 words = [x for x in allwords if len(x) <= i] maxlenwords.append((math.log(len(words), 2), words)) sr = random.SystemRandom() passphrase = [] wordlists_to_use = recursive_subset_sum(entropy, maxlenwords) passphraseentropy = 0.0 for (wle, wl) in wordlists_to_use: passphrase.append(sr.choice(wl)) passphraseentropy += wle return (u".".join(passphrase), passphraseentropy) def main(): parser = argparse.ArgumentParser(prog="passphrase", description="Create a random passphrase by picking a few random words.") parser.add_argument('-d', '--dictionary', help="what file to read a list of words from (or omit this option to use passphrase's bundled dictionary)", type=argparse.FileType('rU'), metavar="DICT") parser.add_argument('bits', help="how many bits of entropy minimum", type=float, metavar="BITS") args = parser.parse_args() dicti = args.dictionary if not dicti: dicti = resource_stream('pyutil', 'data/wordlist.txt') allwords = set([x.decode('utf-8').strip().lower() for x in dicti.readlines()]) passphrase, bits = gen_passphrase(args.bits, allwords) sys.stdout.write(passphrase) sys.stdout.write('\n') sys.stderr.write(u"This passphrase encodes about {:.0f} bits.\n".format(bits)) pyutil-3.3.2/pyutil/scripts/randcookie.py000077500000000000000000000012271437014040400205540ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function import os, sys import zbase32 def main(): if len(sys.argv) > 1: l = int(sys.argv[1]) else: l = 64 bl = (l + 7) / 8 s = zbase32.b2a_l(os.urandom(bl), l) # insert some hyphens for easier memorization chs = 3 + (len(s)%8==0) i = chs while i < len(s)-1: s = s[:i] + "-" + s[i:] i += 1 chs = 7-chs i += chs print(s) if __name__ == '__main__': main() pyutil-3.3.2/pyutil/scripts/randfile.py000077500000000000000000000040761437014040400202270ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import os, sys from random import randrange import argparse def main(): CHUNKSIZE=2**20 parser = argparse.ArgumentParser(prog="randfile", description="Create a file of pseudorandom bytes (not cryptographically secure).") parser.add_argument('-b', '--num-bytes', help="how many bytes to write per output file (default 20)", type=int, metavar="BYTES", default=20) parser.add_argument('-f', '--output-file-prefix', help="prefix of the name of the output file to create and fill with random bytes (default \"randfile\"", metavar="OUTFILEPRE", default="randfile") parser.add_argument('-n', '--num-files', help="how many files to write (default 1)", type=int, metavar="FILES", default=1) parser.add_argument('-F', '--force', help='overwrite any file already present', action='store_true') parser.add_argument('-p', '--progress', help='write an "x" for every file completed and a "." for every %d bytes' % CHUNKSIZE, action='store_true') args = parser.parse_args() for i in xrange(args.num_files): bytesleft = args.num_bytes outputfname = args.output_file_prefix + "." + str(i) if args.force: f = open(outputfname, "wb") else: flags = os.O_WRONLY|os.O_CREAT|os.O_EXCL | (hasattr(os, 'O_BINARY') and os.O_BINARY) fd = os.open(outputfname, flags) f = os.fdopen(fd, "wb") zs = [0]*CHUNKSIZE ts = [256]*CHUNKSIZE while bytesleft >= CHUNKSIZE: f.write(''.join(map(chr, map(randrange, zs, ts)))) bytesleft -= CHUNKSIZE if args.progress: sys.stdout.write(".") ; sys.stdout.flush() zs = [0]*bytesleft ts = [256]*bytesleft f.write(''.join(map(chr, map(randrange, zs, ts)))) if args.progress: sys.stdout.write("x") ; sys.stdout.flush() if __name__ == "__main__": main() pyutil-3.3.2/pyutil/scripts/tailx.py000077500000000000000000000012561437014040400175610ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # output all but the first N lines of a file # Allen Short and Jp Calderone wrote this coool version: import itertools, sys def main(): K = int(sys.argv[1]) if len(sys.argv) > 2: fname = sys.argv[2] inf = open(fname, 'r') else: inf = sys.stdin sys.stdout.writelines(itertools.islice(inf, K, None)) if __name__ == '__main__': main() # thus replacing my dumb version: # # from the Python Standard Library # import sys # # i = K # for l in sys.stdin.readlines(): # if i: # i -= 1 # else: # print l, pyutil-3.3.2/pyutil/scripts/try_decoding.py000066400000000000000000000064451437014040400211140ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function import binascii, codecs, encodings, locale, os, sys, zlib import argparse def listcodecs(dir): names = [] for filename in os.listdir(dir): if filename[-3:] != '.py': continue name = filename[:-3] # Check whether we've found a true codec try: codecs.lookup(name) except LookupError: # Codec not found continue except Exception: # Probably an error from importing the codec; still it's # a valid code name pass names.append(name) return names def listem(): return listcodecs(encodings.__path__[0]) def _canonical_encoding(encoding): if encoding is None: encoding = 'utf-8' encoding = encoding.lower() if encoding == "cp65001": encoding = 'utf-8' elif encoding == "us-ascii" or encoding == "646": encoding = 'ascii' # sometimes Python returns an encoding name that it doesn't support for conversion # fail early if this happens try: u"test".encode(encoding) except (LookupError, AttributeError): raise AssertionError("The character encoding '%s' is not supported for conversion." % (encoding,)) return encoding def get_output_encoding(): return _canonical_encoding(sys.stdout.encoding or locale.getpreferredencoding()) def get_argv_encoding(): if sys.platform == 'win32': # Unicode arguments are not supported on Windows yet; see Tahoe-LAFS tickets #565 and #1074. return 'ascii' else: return get_output_encoding() output_encoding = get_output_encoding() argv_encoding = get_argv_encoding() def type_unicode(argstr): return argstr.decode(argv_encoding) def main(): parser = argparse.ArgumentParser(prog="try_decoding", description="Try decoding some bytes with all sorts of different codecs and print out any that decode.") parser.add_argument('inputfile', help='file to decode or "-" for stdin', type=argparse.FileType('rb'), metavar='INF') parser.add_argument('-t', '--target', help='unicode string to match against (if any)', type=type_unicode, metavar='T') parser.add_argument('-a', '--accept-bytes', help='include codecs which return bytes instead of returning unicode (they will be marked with "!!!" in the output)', action='store_true') args = parser.parse_args() inb = args.inputfile.read() for codec in listem(): try: u = inb.decode(codec) except (UnicodeDecodeError, IOError, TypeError, IndexError, UnicodeError, ValueError, zlib.error, binascii.Error): pass else: if isinstance(u, unicode): if args.target: if args.target != u: continue print("%19s" % codec) print(':') print(u.encode(output_encoding)) else: if not args.accept_bytes: continue print("%19s" % codec) print("!!! ") print(':') print(u) if __name__ == "__main__": main() pyutil-3.3.2/pyutil/scripts/unsort.py000066400000000000000000000007711437014040400177700ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. # randomize the lines of stdin or a file import random, sys def main(): if len(sys.argv) > 1: fname = sys.argv[1] inf = open(fname, 'r') else: inf = sys.stdin lines = inf.readlines() random.shuffle(lines) sys.stdout.writelines(lines) if __name__ == '__main__': main() pyutil-3.3.2/pyutil/scripts/verinfo.py000066400000000000000000000015221437014040400201010ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from __future__ import print_function import exceptions class UsageError(exceptions.Exception): pass import sys import pkg_resources def main(): if len(sys.argv) <= 1: raise UsageError("USAGE: verinfo DISTRIBUTIONNAME [PACKAGENAME]") DISTNAME=sys.argv[1] if len(sys.argv) >= 3: PACKNAME=sys.argv[2] else: PACKNAME=DISTNAME print("pkg_resources.require('%s') => " % (DISTNAME,)) print(pkg_resources.require(DISTNAME)) print("import %s;print %s => " % (PACKNAME, PACKNAME,)) x = __import__(PACKNAME) print(x) print("import %s;print %s.__version__ => " % (PACKNAME, PACKNAME,)) print(hasattr(x, '__version__') and x.__version__) if __name__ == "__main__": main() pyutil-3.3.2/pyutil/strutil.py000066400000000000000000000021741437014040400164540ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. def commonprefix(l): cp = [] for i in range(min(map(len, l))): c = l[0][i] for s in l[1:]: if s[i] != c: return ''.join(cp) cp.append(c) return ''.join(cp) def commonsuffix(l): cp = [] for i in range(min(map(len, l))): c = l[0][-i-1] for s in l[1:]: if s[-i-1] != c: cp.reverse() return ''.join(cp) cp.append(c) cp.reverse() return ''.join(cp) def split_on_newlines(s): """ Splits s on all of the three newline sequences: "\r\n", "\r", or "\n". """ res = [] for x in s.split('\r\n'): for y in x.split('\r'): res.extend(y.split('\n')) return res def pop_trailing_newlines(s): """ @return a copy of s minus any trailing "\n"'s or "\r"'s """ i = len(s)-1 if i < 0: return '' while s[i] in ('\n', '\r',): i = i - 1 if i < 0: return '' return s[:i+1] pyutil-3.3.2/pyutil/test/000077500000000000000000000000001437014040400153475ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/__init__.py000066400000000000000000000000001437014040400174460ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/current/000077500000000000000000000000001437014040400170315ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/current/__init__.py000066400000000000000000000000001437014040400211300ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/current/json_tests/000077500000000000000000000000001437014040400212245ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/current/json_tests/__init__.py000066400000000000000000000000001437014040400233230ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/current/json_tests/test_decode.py000066400000000000000000000010361437014040400240600ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- import decimal from unittest import TestCase from pyutil import jsonutil as json class TestDecode(TestCase): def test_decimal(self): rval = json.loads('1.1', parse_float=decimal.Decimal) self.assert_(isinstance(rval, decimal.Decimal)) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): rval = json.loads('1', parse_int=float) self.assert_(isinstance(rval, float)) self.assertEqual(rval, 1.0) pyutil-3.3.2/pyutil/test/current/json_tests/test_default.py000066400000000000000000000004601437014040400242610ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json class TestDefault(TestCase): def test_default(self): self.assertEqual( json.dumps(type, default=repr), json.dumps(repr(type))) pyutil-3.3.2/pyutil/test/current/json_tests/test_dump.py000066400000000000000000000007051437014040400236040ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase try: from cStringIO import StringIO except ImportError: from io import StringIO from pyutil import jsonutil as json class TestDump(TestCase): def test_dump(self): sio = StringIO() json.dump({}, sio) self.assertEqual(sio.getvalue(), '{}') def test_dumps(self): self.assertEqual(json.dumps({}), '{}') pyutil-3.3.2/pyutil/test/current/json_tests/test_encode_basestring_ascii.py000066400000000000000000000035441437014040400274710ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- import sys from twisted.trial.unittest import SkipTest, TestCase from pyutil.jsonutil import encoder CASES = [ (u'/\\"\ucafe\ubabe\uab98\ufcde\ubcda\uef4a\x08\x0c\n\r\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?', '"/\\\\\\"\\ucafe\\ubabe\\uab98\\ufcde\\ubcda\\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?"'), (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), (u'controls', '"controls"'), (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), (u'{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (u' s p a c e d ', '" s p a c e d "'), (u'\U0001d120', '"\\ud834\\udd20"'), (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), (b'\xce\xb1\xce\xa9', '"\\u03b1\\u03a9"'), (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), (b'\xce\xb1\xce\xa9', '"\\u03b1\\u03a9"'), (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), (u'\u03b1\u03a9', '"\\u03b1\\u03a9"'), (u"`1~!@#$%^&*()_+-={':[,]}|;.?", '"`1~!@#$%^&*()_+-={\':[,]}|;.?"'), (u'\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), (u'\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBaseStringAscii(TestCase): def test_py_encode_basestring_ascii(self): self._test_encode_basestring_ascii(encoder.py_encode_basestring_ascii) def test_c_encode_basestring_ascii(self): if not encoder.c_encode_basestring_ascii: raise SkipTest("no C extension speedups available to test") self._test_encode_basestring_ascii(encoder.c_encode_basestring_ascii) def _test_encode_basestring_ascii(self, encode_basestring_ascii): for input_string, expect in CASES: result = encode_basestring_ascii(input_string) self.assertEqual(result, expect) pyutil-3.3.2/pyutil/test/current/json_tests/test_fail.py000066400000000000000000000056451437014040400235620ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json # Fri Dec 30 18:57:26 2005 JSONDOCS = [ # http://json.org/JSON_checker/test/fail1.json '"A JSON payload should be an object or array, not a string."', # http://json.org/JSON_checker/test/fail2.json '["Unclosed array"', # http://json.org/JSON_checker/test/fail3.json '{unquoted_key: "keys must be quoted}', # http://json.org/JSON_checker/test/fail4.json '["extra comma",]', # http://json.org/JSON_checker/test/fail5.json '["double extra comma",,]', # http://json.org/JSON_checker/test/fail6.json '[ , "<-- missing value"]', # http://json.org/JSON_checker/test/fail7.json '["Comma after the close"],', # http://json.org/JSON_checker/test/fail8.json '["Extra close"]]', # http://json.org/JSON_checker/test/fail9.json '{"Extra comma": true,}', # http://json.org/JSON_checker/test/fail10.json '{"Extra value after close": true} "misplaced quoted value"', # http://json.org/JSON_checker/test/fail11.json '{"Illegal expression": 1 + 2}', # http://json.org/JSON_checker/test/fail12.json '{"Illegal invocation": alert()}', # http://json.org/JSON_checker/test/fail13.json '{"Numbers cannot have leading zeroes": 013}', # http://json.org/JSON_checker/test/fail14.json '{"Numbers cannot be hex": 0x14}', # http://json.org/JSON_checker/test/fail15.json '["Illegal backslash escape: \\x15"]', # http://json.org/JSON_checker/test/fail16.json '["Illegal backslash escape: \\\'"]', # http://json.org/JSON_checker/test/fail17.json '["Illegal backslash escape: \\017"]', # http://json.org/JSON_checker/test/fail18.json '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', # http://json.org/JSON_checker/test/fail19.json '{"Missing colon" null}', # http://json.org/JSON_checker/test/fail20.json '{"Double colon":: null}', # http://json.org/JSON_checker/test/fail21.json '{"Comma instead of colon", null}', # http://json.org/JSON_checker/test/fail22.json '["Colon instead of comma": false]', # http://json.org/JSON_checker/test/fail23.json '["Bad value", truth]', # http://json.org/JSON_checker/test/fail24.json "['single quote']", # http://code.google.com/p/simplejson/issues/detail?id=3 u'["A\u001FZ control characters in string"]', ] SKIPS = { 1: "why not have a string payload?", 18: "spec doesn't specify any nesting limitations", } class TestFail(TestCase): def test_failures(self): for idx, doc in enumerate(JSONDOCS): idx = idx + 1 if idx in SKIPS: json.loads(doc) continue try: json.loads(doc) except ValueError: pass else: self.fail("Expected failure for fail%d.json: %r" % (idx, doc)) pyutil-3.3.2/pyutil/test/current/json_tests/test_float.py000066400000000000000000000005301437014040400237400ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- import math from unittest import TestCase from pyutil import jsonutil as json class TestFloat(TestCase): def test_floats(self): for num in [1617161771.7650001, math.pi, math.pi**100, math.pi**-100]: self.assertEqual(float(json.dumps(num)), num) pyutil-3.3.2/pyutil/test/current/json_tests/test_indent.py000066400000000000000000000017501437014040400241210ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json import textwrap class TestIndent(TestCase): def test_indent(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] expect = textwrap.dedent("""\ [ [ "blorpie" ], [ "whoops" ], [], "d-shtaeou", "d-nthiouh", "i-vhbjkhnth", { "nifty": 87 }, { "field": "yes", "morefield": false } ]""") d1 = json.dumps(h) d2 = json.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) h1 = json.loads(d1) h2 = json.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) pyutil-3.3.2/pyutil/test/current/json_tests/test_pass1.py000066400000000000000000000035361437014040400236730ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json # from http://json.org/JSON_checker/test/pass1.json JSON = r''' [ "JSON Test Pattern pass1", {"object with 1 member":["array with 1 element"]}, {}, [], -42, true, false, null, { "integer": 1234567890, "real": -9876.543210, "e": 0.123456789e-12, "E": 1.234567890E+34, "": 23456789012E666, "zero": 0, "one": 1, "space": " ", "quote": "\"", "backslash": "\\", "controls": "\b\f\n\r\t", "slash": "/ & \/", "alpha": "abcdefghijklmnopqrstuvwyz", "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", "digit": "0123456789", "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", "true": true, "false": false, "null": null, "array":[ ], "object":{ }, "address": "50 St. James Street", "url": "http://www.JSON.org/", "comment": "// /* */": " ", " s p a c e d " :[1,2 , 3 , 4 , 5 , 6 ,7 ], "compact": [1,2,3,4,5,6,7], "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", "quotes": "" \u0022 %22 0x22 034 "", "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" : "A key can be any string" }, 0.5 ,98.6 , 99.44 , 1066 ,"rosebud"] ''' class TestPass1(TestCase): def test_parse(self): # test in/out equivalence and parsing res = json.loads(JSON) out = json.dumps(res) self.assertEqual(res, json.loads(out)) self.assertTrue("2.3456789012E+676" in json.dumps(res, allow_nan=False)) pyutil-3.3.2/pyutil/test/current/json_tests/test_pass2.py000066400000000000000000000007251437014040400236710ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json # from http://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' class TestPass2(TestCase): def test_parse(self): # test in/out equivalence and parsing res = json.loads(JSON) out = json.dumps(res) self.assertEqual(res, json.loads(out)) pyutil-3.3.2/pyutil/test/current/json_tests/test_pass3.py000066400000000000000000000010651437014040400236700ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json # from http://json.org/JSON_checker/test/pass3.json JSON = r''' { "JSON Test Pattern pass3": { "The outermost value": "must be an object or array.", "In this test": "It is an object." } } ''' class TestPass3(TestCase): def test_parse(self): # test in/out equivalence and parsing res = json.loads(JSON) out = json.dumps(res) self.assertEqual(res, json.loads(out)) pyutil-3.3.2/pyutil/test/current/json_tests/test_recursion.py000066400000000000000000000033361437014040400246530ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json class JSONTestObject: pass class RecursiveJSONEncoder(json.JSONEncoder): recurse = False def default(self, o): if o is JSONTestObject: if self.recurse: return [JSONTestObject] else: return 'JSONTestObject' return json.JSONEncoder.default(o) class TestRecursion(TestCase): def test_listrecursion(self): x = [] x.append(x) try: json.dumps(x) except ValueError: pass else: self.fail("didn't raise ValueError on list recursion") x = [] y = [x] x.append(y) try: json.dumps(x) except ValueError: pass else: self.fail("didn't raise ValueError on alternating list recursion") y = [] x = [y, y] # ensure that the marker is cleared json.dumps(x) def test_dictrecursion(self): x = {} x["test"] = x try: json.dumps(x) except ValueError: pass else: self.fail("didn't raise ValueError on dict recursion") x = {} {"a": x, "b": x} # ensure that the marker is cleared json.dumps(x) def test_defaultrecursion(self): enc = RecursiveJSONEncoder() self.assertEqual(enc.encode(JSONTestObject), '"JSONTestObject"') enc.recurse = True try: enc.encode(JSONTestObject) except ValueError: pass else: self.fail("didn't raise ValueError on default recursion") pyutil-3.3.2/pyutil/test/current/json_tests/test_separators.py000066400000000000000000000017761437014040400250330ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- import textwrap from unittest import TestCase from pyutil import jsonutil as json class TestSeparators(TestCase): def test_separators(self): h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', {'nifty': 87}, {'field': 'yes', 'morefield': False} ] expect = textwrap.dedent("""\ [ [ "blorpie" ] , [ "whoops" ] , [] , "d-shtaeou" , "d-nthiouh" , "i-vhbjkhnth" , { "nifty" : 87 } , { "field" : "yes" , "morefield" : false } ]""") d1 = json.dumps(h) d2 = json.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) h1 = json.loads(d1) h2 = json.loads(d2) self.assertEqual(h1, h) self.assertEqual(h2, h) self.assertEqual(d2, expect) pyutil-3.3.2/pyutil/test/current/json_tests/test_speedups.py000066400000000000000000000016121437014040400244650ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from twisted.trial.unittest import SkipTest, TestCase from pyutil.jsonutil import decoder from pyutil.jsonutil import encoder class TestSpeedups(TestCase): def test_scanstring(self): if not encoder.c_encode_basestring_ascii: raise SkipTest("no C extension speedups available to test") self.assertEqual(decoder.scanstring.__module__, "simplejson._speedups") self.assert_(decoder.scanstring is decoder.c_scanstring) def test_encode_basestring_ascii(self): if not encoder.c_encode_basestring_ascii: raise SkipTest("no C extension speedups available to test") self.assertEqual(encoder.encode_basestring_ascii.__module__, "simplejson._speedups") self.assert_(encoder.encode_basestring_ascii is encoder.c_encode_basestring_ascii) pyutil-3.3.2/pyutil/test/current/json_tests/test_unicode.py000066400000000000000000000040601437014040400242630ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from unittest import TestCase from pyutil import jsonutil as json try: unichr = unichr except NameError: unichr = chr class TestUnicode(TestCase): def test_encoding1(self): encoder = json.JSONEncoder(encoding='utf-8') u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') ju = encoder.encode(u) js = encoder.encode(s) self.assertEqual(ju, js) def test_encoding2(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' s = u.encode('utf-8') ju = json.dumps(u, encoding='utf-8') js = json.dumps(s, encoding='utf-8') self.assertEqual(ju, js) def test_encoding3(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = json.dumps(u) self.assertEqual(j, '"\\u03b1\\u03a9"') def test_encoding4(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = json.dumps([u]) self.assertEqual(j, '["\\u03b1\\u03a9"]') def test_encoding5(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = json.dumps(u, ensure_ascii=False) self.assertEqual(j, u'"%s"' % (u,)) def test_encoding6(self): u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = json.dumps([u], ensure_ascii=False) self.assertEqual(j, u'["%s"]' % (u,)) def test_big_unicode_encode(self): u = u'\U0001d120' self.assertEqual(json.dumps(u), '"\\ud834\\udd20"') self.assertEqual(json.dumps(u, ensure_ascii=False), u'"\U0001d120"') def test_big_unicode_decode(self): u = u'z\U0001d120x' self.assertEqual(json.loads('"' + u + '"'), u) self.assertEqual(json.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = unichr(i) js = '"\\u%04x"' % (i,) self.assertEqual(json.loads(js), u) pyutil-3.3.2/pyutil/test/current/test_assertutil.py000066400000000000000000000017101437014040400226400ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. # Python Standard Library modules import sys, unittest from pyutil import assertutil class AssertUtilTestCase(unittest.TestCase): def test_bad_precond(self): adict=23 try: assertutil.precondition(isinstance(adict, dict), "adict is required to be a dict.", 23, adict=adict, foo=None) except AssertionError as le: if sys.version_info[0] == 2: self.assertEqual(le.args[0], "precondition: 'adict is required to be a dict.' , 23 , 'adict': 23 , 'foo': None ") else: self.assertEqual(le.args[0], "precondition: 'adict is required to be a dict.' , 23 , 'adict': 23 , 'foo': None ") pyutil-3.3.2/pyutil/test/current/test_fileutil.py000066400000000000000000000020661437014040400222630ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest import os from pyutil import fileutil class FileUtil(unittest.TestCase): def mkdir(self, basedir, path, mode=0o777): fn = os.path.join(basedir, path) fileutil.make_dirs(fn, mode) def touch(self, basedir, path, mode=None, data="touch\n"): fn = os.path.join(basedir, path) f = open(fn, "w") f.write(data) f.close() if mode is not None: os.chmod(fn, mode) def test_du(self): basedir = "util/FileUtil/test_du" fileutil.make_dirs(basedir) d = os.path.join(basedir, "space-consuming") self.mkdir(d, "a/b") self.touch(d, "a/b/1.txt", data="a"*10) self.touch(d, "a/b/2.txt", data="b"*11) self.mkdir(d, "a/c") self.touch(d, "a/c/1.txt", data="c"*12) self.touch(d, "a/c/2.txt", data="d"*13) used = fileutil.du(basedir) self.assertEqual(10+11+12+13, used) pyutil-3.3.2/pyutil/test/current/test_iputil.py000066400000000000000000000027251437014040400217560ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function try: from twisted.trial import unittest unittest # http://divmod.org/trac/ticket/1499 except ImportError as le: print("Skipping test_iputil since it requires Twisted and Twisted could not be imported: %s" % (le,)) else: from pyutil import iputil, testutil import re DOTTED_QUAD_RE=re.compile("^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$") class ListAddresses(testutil.SignalMixin): def test_get_local_ip_for(self): addr = iputil.get_local_ip_for('127.0.0.1') self.assertTrue(DOTTED_QUAD_RE.match(addr)) def test_list_async(self): try: from twisted.trial import unittest unittest # http://divmod.org/trac/ticket/1499 from pyutil import iputil except ImportError as le: raise unittest.SkipTest("iputil could not be imported (probably because its dependency, Twisted, is not installed). %s" % (le,)) d = iputil.get_local_addresses_async() def _check(addresses): self.assertTrue(len(addresses) >= 1) # always have localhost self.assertTrue("127.0.0.1" in addresses, addresses) d.addCallbacks(_check) return d test_list_async.timeout=2 pyutil-3.3.2/pyutil/test/current/test_jsonutil.py000066400000000000000000000012221437014040400223060ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest from decimal import Decimal from pyutil import jsonutil zero_point_one = Decimal("0.1") class TestDecimal(unittest.TestCase): def test_encode(self): self.assertEqual(jsonutil.dumps(zero_point_one), "0.1") def test_decode(self): self.assertEqual(jsonutil.loads("0.1"), zero_point_one) def test_no_exception_on_convergent_parse_float(self): self.assertEqual(jsonutil.loads("0.1", parse_float=Decimal), zero_point_one) pyutil-3.3.2/pyutil/test/current/test_mathutil.py000066400000000000000000000124331437014040400222740ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest from pyutil import mathutil from pyutil.assertutil import _assert class MathUtilTestCase(unittest.TestCase): def _help_test_is_power_of_k(self, k): for i in range(2, 40): _assert(mathutil.is_power_of_k(k**i, k), k, i) def test_is_power_of_k(self): for i in range(2, 5): self._help_test_is_power_of_k(i) def test_log_ceil(self): f = mathutil.log_ceil self.assertEqual(f(1, 2), 0) self.assertEqual(f(1, 3), 0) self.assertEqual(f(2, 2), 1) self.assertEqual(f(2, 3), 1) self.assertEqual(f(3, 2), 2) def test_log_floor(self): f = mathutil.log_floor self.assertEqual(f(1, 2), 0) self.assertEqual(f(1, 3), 0) self.assertEqual(f(2, 2), 1) self.assertEqual(f(2, 3), 0) self.assertEqual(f(3, 2), 1) def test_div_ceil(self): f = mathutil.div_ceil self.assertEqual(f(0, 1), 0) self.assertEqual(f(0, 2), 0) self.assertEqual(f(0, 3), 0) self.assertEqual(f(1, 3), 1) self.assertEqual(f(2, 3), 1) self.assertEqual(f(3, 3), 1) self.assertEqual(f(4, 3), 2) self.assertEqual(f(5, 3), 2) self.assertEqual(f(6, 3), 2) self.assertEqual(f(7, 3), 3) self.assertTrue(isinstance(f(0.0, 1), int)) self.assertEqual(f(7.0, 3.0), 3) self.assertEqual(f(7, 3.0), 3) self.assertEqual(f(7.0, 3), 3) self.assertEqual(f(6.0, 3.0), 2) self.assertEqual(f(6.0, 3), 2) self.assertEqual(f(6, 3.0), 2) def test_next_multiple(self): f = mathutil.next_multiple self.assertEqual(f(5, 1), 5) self.assertEqual(f(5, 2), 6) self.assertEqual(f(5, 3), 6) self.assertEqual(f(5, 4), 8) self.assertEqual(f(5, 5), 5) self.assertEqual(f(5, 6), 6) self.assertEqual(f(32, 1), 32) self.assertEqual(f(32, 2), 32) self.assertEqual(f(32, 3), 33) self.assertEqual(f(32, 4), 32) self.assertEqual(f(32, 5), 35) self.assertEqual(f(32, 6), 36) self.assertEqual(f(32, 7), 35) self.assertEqual(f(32, 8), 32) self.assertEqual(f(32, 9), 36) self.assertEqual(f(32, 10), 40) self.assertEqual(f(32, 11), 33) self.assertEqual(f(32, 12), 36) self.assertEqual(f(32, 13), 39) self.assertEqual(f(32, 14), 42) self.assertEqual(f(32, 15), 45) self.assertEqual(f(32, 16), 32) self.assertEqual(f(32, 17), 34) self.assertEqual(f(32, 18), 36) self.assertEqual(f(32, 589), 589) def test_pad_size(self): f = mathutil.pad_size self.assertEqual(f(0, 4), 0) self.assertEqual(f(1, 4), 3) self.assertEqual(f(2, 4), 2) self.assertEqual(f(3, 4), 1) self.assertEqual(f(4, 4), 0) self.assertEqual(f(5, 4), 3) def test_is_power_of_k_part_2(self): f = mathutil.is_power_of_k for i in range(1, 100): if i in (1, 2, 4, 8, 16, 32, 64): self.assertTrue(f(i, 2), "but %d *is* a power of 2" % i) else: self.assertFalse(f(i, 2), "but %d is *not* a power of 2" % i) for i in range(1, 100): if i in (1, 3, 9, 27, 81): self.assertTrue(f(i, 3), "but %d *is* a power of 3" % i) else: self.assertFalse(f(i, 3), "but %d is *not* a power of 3" % i) def test_next_power_of_k(self): f = mathutil.next_power_of_k self.assertEqual(f(0,2), 1) self.assertEqual(f(1,2), 1) self.assertEqual(f(2,2), 2) self.assertEqual(f(3,2), 4) self.assertEqual(f(4,2), 4) for i in range(5, 8): self.assertEqual(f(i,2), 8, "%d" % i) for i in range(9, 16): self.assertEqual(f(i,2), 16, "%d" % i) for i in range(17, 32): self.assertEqual(f(i,2), 32, "%d" % i) for i in range(33, 64): self.assertEqual(f(i,2), 64, "%d" % i) for i in range(65, 100): self.assertEqual(f(i,2), 128, "%d" % i) self.assertEqual(f(0,3), 1) self.assertEqual(f(1,3), 1) self.assertEqual(f(2,3), 3) self.assertEqual(f(3,3), 3) for i in range(4, 9): self.assertEqual(f(i,3), 9, "%d" % i) for i in range(10, 27): self.assertEqual(f(i,3), 27, "%d" % i) for i in range(28, 81): self.assertEqual(f(i,3), 81, "%d" % i) for i in range(82, 200): self.assertEqual(f(i,3), 243, "%d" % i) def test_ave(self): f = mathutil.ave self.assertEqual(f([1,2,3]), 2) self.assertEqual(f([0,0,0,4]), 1) self.assertAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666) def assertEqualContents(self, a, b): self.assertEqual(sorted(a), sorted(b)) def test_permute(self): f = mathutil.permute self.assertEqualContents(f([]), []) self.assertEqualContents(f([1]), [[1]]) self.assertEqualContents(f([1,2]), [[1,2], [2,1]]) self.assertEqualContents(f([1,2,3]), [[1,2,3], [1,3,2], [2,1,3], [2,3,1], [3,1,2], [3,2,1]]) pyutil-3.3.2/pyutil/test/current/test_observer.py000066400000000000000000000054421437014040400222760ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from twisted.trial import unittest from twisted.internet import defer, reactor from pyutil import observer def nextTurn(res=None): d = defer.Deferred() reactor.callLater(0, d.callback, res) return d class Observer(unittest.TestCase): def test_oneshot(self): ol = observer.OneShotObserverList() rep = repr(ol) self.assertEqual(rep, "") d1 = ol.when_fired() d2 = ol.when_fired() def _addmore(res): self.assertEqual(res, "result") d3 = ol.when_fired() d3.addCallback(self.assertEqual, "result") return d3 d1.addCallback(_addmore) ol.fire("result") rep = repr(ol) self.assertEqual(rep, " result>") d4 = ol.when_fired() dl = defer.DeferredList([d1,d2,d4]) return dl def test_oneshot_fireagain(self): ol = observer.OneShotObserverList() d = ol.when_fired() def _addmore(res): self.assertEqual(res, "result") ol.fire_if_not_fired("result3") # should be ignored d2 = ol.when_fired() d2.addCallback(self.assertEqual, "result") return d2 d.addCallback(_addmore) ol.fire_if_not_fired("result") ol.fire_if_not_fired("result2") return d def test_lazy_oneshot(self): ol = observer.LazyOneShotObserverList() d1 = ol.when_fired() d2 = ol.when_fired() def _addmore(res): self.assertEqual(res, "result") d3 = ol.when_fired() d3.addCallback(self.assertEqual, "result") return d3 d1.addCallback(_addmore) def _get_result(): return "result" ol.fire(_get_result) d4 = ol.when_fired() dl = defer.DeferredList([d1,d2,d4]) return dl def test_observerlist(self): ol = observer.ObserverList() l1 = [] l2 = [] l3 = [] ol.subscribe(l1.append) ol.notify(1) ol.subscribe(l2.append) ol.notify(2) ol.unsubscribe(l1.append) ol.notify(3) def _check(res): self.assertEqual(l1, [1,2]) self.assertEqual(l2, [2,3]) d = nextTurn() d.addCallback(_check) def _step2(res): def _add(a, b, c=None): l3.append((a,b,c)) ol.unsubscribe(l2.append) ol.subscribe(_add) ol.notify(4, 5, c=6) return nextTurn() def _check2(res): self.assertEqual(l3, [(4,5,6)]) d.addCallback(_step2) d.addCallback(_check2) return d pyutil-3.3.2/pyutil/test/current/test_time_format.py000066400000000000000000000077561437014040400227670ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """\ Test time_format.py """ import os, time, unittest from pyutil import time_format, increasing_timer class TimeUtilTestCase(unittest.TestCase): def setUp(self): pass def tearDown(self): pass def test_iso8601_utc_time(self, timer=increasing_timer.timer): ts1 = time_format.iso_utc(timer.time() - 20) ts2 = time_format.iso_utc() assert ts1 < ts2, "failed: %s < %s" % (ts1, ts2) ts3 = time_format.iso_utc(timer.time() + 20) assert ts2 < ts3, "failed: %s < %s" % (ts2, ts3) def test_iso_utc_time_to_localseconds(self, timer=increasing_timer.timer): # test three times of the year so that a DST problem would hopefully be triggered t1 = int(timer.time() - 365*3600/3) iso_utc_t1 = time_format.iso_utc(t1) t1_2 = time_format.iso_utc_time_to_seconds(iso_utc_t1) assert t1 == t1_2, (t1, t1_2) t1 = int(timer.time() - (365*3600*2/3)) iso_utc_t1 = time_format.iso_utc(t1) t1_2 = time_format.iso_utc_time_to_seconds(iso_utc_t1) self.assertEqual(t1, t1_2) t1 = int(timer.time()) iso_utc_t1 = time_format.iso_utc(t1) t1_2 = time_format.iso_utc_time_to_seconds(iso_utc_t1) self.assertEqual(t1, t1_2) def test_epoch(self): return self._help_test_epoch() def test_epoch_in_London(self): # Europe/London is a particularly troublesome timezone. Nowadays, its # offset from GMT is 0. But in 1970, its offset from GMT was 1. # (Apparently in 1970 Britain had redefined standard time to be GMT+1 # and stayed in standard time all year round, whereas today # Europe/London standard time is GMT and Europe/London Daylight # Savings Time is GMT+1.) The current implementation of # time_format.iso_utc_time_to_seconds() breaks if the timezone is # Europe/London. (As soon as this unit test is done then I'll change # that implementation to something that works even in this case...) origtz = os.environ.get('TZ') os.environ['TZ'] = "Europe/London" if hasattr(time, 'tzset'): time.tzset() try: return self._help_test_epoch() finally: if origtz is None: del os.environ['TZ'] else: os.environ['TZ'] = origtz if hasattr(time, 'tzset'): time.tzset() def _help_test_epoch(self): origtzname = time.tzname s = time_format.iso_utc_time_to_seconds("1970-01-01T00:00:01Z") self.assertEqual(s, 1.0) s = time_format.iso_utc_time_to_seconds("1970-01-01_00:00:01Z") self.assertEqual(s, 1.0) s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01Z") self.assertEqual(s, 1.0) self.assertEqual(time_format.iso_utc(1.0), "1970-01-01 00:00:01Z") self.assertEqual(time_format.iso_utc(1.0, sep="_"), "1970-01-01_00:00:01Z") now = time.time() isostr = time_format.iso_utc(now) timestamp = time_format.iso_utc_time_to_seconds(isostr) self.assertEqual(int(timestamp), int(now)) def my_time(): return 1.0 self.assertEqual(time_format.iso_utc(t=my_time), "1970-01-01 00:00:01Z") self.assertRaises(ValueError, time_format.iso_utc_time_to_seconds, "invalid timestring") s = time_format.iso_utc_time_to_seconds("1970-01-01 00:00:01.500Z") self.assertEqual(s, 1.5) # Look for daylight-savings-related errors. thatmomentinmarch = time_format.iso_utc_time_to_seconds("2009-03-20 21:49:02.226536Z") self.assertEqual(thatmomentinmarch, 1237585742.226536) self.assertEqual(origtzname, time.tzname) pyutil-3.3.2/pyutil/test/current/test_verlib.py000066400000000000000000000104741437014040400217330ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """Tests for distutils.version.""" import unittest import doctest from pyutil.verlib import NormalizedVersion as V from pyutil.verlib import IrrationalVersionError from pyutil.verlib import suggest_normalized_version as suggest class VersionTestCase(unittest.TestCase): versions = ((V('1.0'), '1.0'), (V('1.1'), '1.1'), (V('1.2.3'), '1.2.3'), (V('1.2'), '1.2'), (V('1.2.3a4'), '1.2.3a4'), (V('1.2c4'), '1.2c4'), (V('1.2.3.4'), '1.2.3.4'), (V('1.2.3.4.0b3'), '1.2.3.4b3'), (V('1.2.0.0.0'), '1.2'), (V('1.0.dev345'), '1.0.dev345'), (V('1.0.post456.dev623'), '1.0.post456.dev623')) def test_basic_versions(self): for v, s in self.versions: self.assertEqual(str(v), s) def test_from_parts(self): for v, s in self.versions: v2 = V.from_parts(*v.parts) self.assertEqual(v, v2) self.assertEqual(str(v), str(v2)) def test_irrational_versions(self): irrational = ('1', '1.2a', '1.2.3b', '1.02', '1.2a03', '1.2a3.04', '1.2.dev.2', '1.2dev', '1.2.dev', '1.2.dev2.post2', '1.2.post2.dev3.post4') for s in irrational: self.assertRaises(IrrationalVersionError, V, s) def test_comparison(self): r""" >>> V('1.2.0') == '1.2' Traceback (most recent call last): ... TypeError: cannot compare NormalizedVersion and str >>> V('1.2.0') == V('1.2') True >>> V('1.2.0') == V('1.2.3') False >>> V('1.2.0') < V('1.2.3') True >>> (V('1.0') > V('1.0b2')) True >>> (V('1.0') > V('1.0c2') > V('1.0c1') > V('1.0b2') > V('1.0b1') ... > V('1.0a2') > V('1.0a1')) True >>> (V('1.0.0') > V('1.0.0c2') > V('1.0.0c1') > V('1.0.0b2') > V('1.0.0b1') ... > V('1.0.0a2') > V('1.0.0a1')) True >>> V('1.0') < V('1.0.post456.dev623') True >>> V('1.0.post456.dev623') < V('1.0.post456') < V('1.0.post1234') True >>> (V('1.0a1') ... < V('1.0a2.dev456') ... < V('1.0a2') ... < V('1.0a2.1.dev456') # e.g. need to do a quick post release on 1.0a2 ... < V('1.0a2.1') ... < V('1.0b1.dev456') ... < V('1.0b2') ... < V('1.0c1.dev456') ... < V('1.0c1') ... < V('1.0.dev7') ... < V('1.0.dev18') ... < V('1.0.dev456') ... < V('1.0.dev1234') ... < V('1.0') ... < V('1.0.post456.dev623') # development version of a post release ... < V('1.0.post456')) True """ # must be a simpler way to call the docstrings doctest.run_docstring_examples(self.test_comparison, globals(), name='test_comparison') def test_suggest_normalized_version(self): self.assertEqual(suggest('1.0'), '1.0') self.assertEqual(suggest('1.0-alpha1'), '1.0a1') self.assertEqual(suggest('1.0c2'), '1.0c2') self.assertEqual(suggest('walla walla washington'), None) self.assertEqual(suggest('2.4c1'), '2.4c1') # from setuptools self.assertEqual(suggest('0.4a1.r10'), '0.4a1.post10') self.assertEqual(suggest('0.7a1dev-r66608'), '0.7a1.dev66608') self.assertEqual(suggest('0.6a9.dev-r41475'), '0.6a9.dev41475') self.assertEqual(suggest('2.4preview1'), '2.4c1') self.assertEqual(suggest('2.4pre1') , '2.4c1') self.assertEqual(suggest('2.1-rc2'), '2.1c2') # from pypi self.assertEqual(suggest('0.1dev'), '0.1.dev0') self.assertEqual(suggest('0.1.dev'), '0.1.dev0') # we want to be able to parse Twisted # development versions are like post releases in Twisted self.assertEqual(suggest('9.0.0+r2363'), '9.0.0.post2363') # pre-releases are using markers like "pre1" self.assertEqual(suggest('9.0.0pre1'), '9.0.0c1') # we want to be able to parse Tcl-TK # they us "p1" "p2" for post releases self.assertEqual(suggest('1.4p1'), '1.4.post1') pyutil-3.3.2/pyutil/test/current/test_version_class.py000066400000000000000000000017441437014040400233220ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest from pyutil import version_class V = version_class.Version class T(unittest.TestCase): def test_rc_regex_rejects_rc_suffix(self): self.assertRaises(ValueError, V, '9.9.9rc9') def test_rc_regex_rejects_trailing_garbage(self): self.assertRaises(ValueError, V, '9.9.9c9HEYTHISISNTRIGHT') def test_comparisons(self): self.assertTrue(V('1.0') < V('1.1')) self.assertTrue(V('1.0a1') < V('1.0')) self.assertTrue(V('1.0a1') < V('1.0b1')) self.assertTrue(V('1.0b1') < V('1.0c1')) self.assertTrue(V('1.0a1') < V('1.0a1-r99')) self.assertEqual(V('1.0a1.post987'), V('1.0a1-r987')) self.assertEqual(str(V('1.0a1.post999')), '1.0.0a1-r999') self.assertEqual(str(V('1.0a1-r999')), '1.0.0a1-r999') self.assertNotEqual(V('1.0a1'), V('1.0a1-r987')) pyutil-3.3.2/pyutil/test/deprecated/000077500000000000000000000000001437014040400174475ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/deprecated/__init__.py000066400000000000000000000000001437014040400215460ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/deprecated/test_dictutil.py000066400000000000000000000070041437014040400227020ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import random, sys, traceback, unittest from pyutil.assertutil import _assert from pyutil import dictutil class EqButNotIs: def __init__(self, x): self.x = x self.hash = int(random.randrange(0, 2**31)) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.x,) def __hash__(self): return self.hash def __le__(self, other): return self.x <= other def __lt__(self, other): return self.x < other def __ge__(self, other): return self.x >= other def __gt__(self, other): return self.x > other def __ne__(self, other): return self.x != other def __eq__(self, other): return self.x == other class Testy(unittest.TestCase): def _help_test_empty_dict(self, klass): d1 = klass() d2 = klass({}) self.assertTrue(d1 == d2, "klass: %s, d1: %r, d2: %r" % (klass, d1, d2,)) self.assertTrue(len(d1) == 0) self.assertTrue(len(d2) == 0) def _help_test_nonempty_dict(self, klass): # Python 2 allowed comparison between str and int, # therefore mixing values of different types in ValueOrderedDict # would work. It's now a TypeError in Python 3. #d1 = klass({'a': 1, 'b': "eggs", 3: "spam",}) d1 = klass({'a': '1', 'b': "eggs", 3: "spam",}) d2 = klass({'a': '1', 'b': "eggs", 3: "spam",}) self.assertTrue(d1 == d2) self.assertTrue(len(d1) == 3, "%s, %s" % (len(d1), d1,)) self.assertTrue(len(d2) == 3) def _help_test_eq_but_notis(self, klass): d = klass({'a': 3, 'b': EqButNotIs(3), 'c': 3}) d.pop('b') d.clear() d['a'] = 3 d['b'] = EqButNotIs(3) d['c'] = 3 d.pop('b') d.clear() d['b'] = EqButNotIs(3) d['a'] = 3 d['c'] = 3 d.pop('b') d.clear() d['a'] = EqButNotIs(3) d['c'] = 3 d['a'] = 3 d.clear() fake3 = EqButNotIs(3) fake7 = EqButNotIs(7) d[fake3] = fake7 d[3] = 7 d[3] = 8 _assert(any(x for x in d.values() if x is 8)) _assert(any(x for x in d.values() if x is fake7)) _assert(not any(x for x in d.values() if x is 7)) # The real 7 should have been ejected by the d[3] = 8. _assert(any(x for x in d if x is fake3)) _assert(any(x for x in d if x is 3)) d[fake3] = 8 d.clear() d[3] = 7 fake3 = EqButNotIs(3) fake7 = EqButNotIs(7) d[fake3] = fake7 d[3] = 8 _assert(any(x for x in d.values() if x is 8)) _assert(any(x for x in d.values() if x is fake7)) _assert(not any(x for x in d.values() if x is 7)) # The real 7 should have been ejected by the d[3] = 8. _assert(any(x for x in d if x is fake3)) _assert(any(x for x in d if x is 3)) d[fake3] = 8 def test_em(self): for klass in (dictutil.UtilDict, dictutil.NumDict, dictutil.ValueOrderedDict,): # print "name of class: ", klass for helper in (self._help_test_empty_dict, self._help_test_nonempty_dict, self._help_test_eq_but_notis,): # print "name of test func: ", helper helper(klass) def suite(): suite = unittest.makeSuite(Testy, 'test') return suite if __name__ == '__main__': unittest.main() pyutil-3.3.2/pyutil/test/deprecated/test_picklesaver.py000066400000000000000000000025471437014040400234000ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function import os try: from twisted.trial import unittest except ImportError as le: print("Skipping %s since it requires Twisted and Twisted could not be imported: %s" % (__name__, le,)) else: from pyutil import PickleSaver, fileutil class Thingie(PickleSaver.PickleSaver): def __init__(self, fname, delay=30): PickleSaver.PickleSaver.__init__(self, fname=fname, attrs={'tmp_store':'False'}, DELAY=delay) class PickleSaverTest(unittest.TestCase): def _test_save_now(self, fname): thingie = Thingie(fname, delay=0) thingie.tmp_store = 'True' thingie.lazy_save() # Note: it was constructed with default save delay of 0. def test_save_now(self): """ This test should create a lazy save object, save it with no delay and check if the file exists. """ tempdir = fileutil.NamedTemporaryDirectory() fname = os.path.join(tempdir.name, "picklesavertest") self._test_save_now(fname) self.assertTrue(os.path.isfile(fname), "The file [%s] does not exist." %(fname,)) tempdir.shutdown() pyutil-3.3.2/pyutil/test/deprecated/test_xor.py000066400000000000000000000013521437014040400216710ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest from pyutil.xor import xor # unit tests def _help_test(xf): assert xf(b'\000', b'\000') == b'\000' assert xf(b'\001', b'\000') == b'\001' assert xf(b'\001', b'\001') == b'\000' assert xf(b'\000\001', b'\000\001') == b'\000\000' assert xf(b'\100\101', b'\000\101') == b'\100\000' class Testy(unittest.TestCase): def test_em(self): for xorfunc in (xor.py_xor, xor.py_xor_simple, xor.xor,): if callable(xorfunc): # print "testing xorfunc ", xorfunc _help_test(xorfunc) pyutil-3.3.2/pyutil/test/out_of_shape/000077500000000000000000000000001437014040400200225ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/out_of_shape/__init__.py000066400000000000000000000000001437014040400221210ustar00rootroot00000000000000pyutil-3.3.2/pyutil/test/out_of_shape/test_cache.py000066400000000000000000000412261437014040400225030ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function import random, unittest from pyutil.assertutil import _assert from pyutil.humanreadable import hr from pyutil import memutil from pyutil import cache class Bencher: def __init__(self, klass, MAXREPS=2**8, MAXTIME=5): print(klass) self.klass = klass self.MAXREPS = MAXREPS self.MAXTIME = MAXTIME self.d = {} self.lrun = None def _generic_benchmarking_init(self, n): self.d.clear() global lrun self.lrun = self.klass(maxsize=n) for i in range(n): self.d[i] = i self.lrun[n+i] = n+i def _benchmark_init(self, n): MAXSIZE=n/2 d2 = self.klass(initialdata=self.d, maxsize=MAXSIZE) assert len(d2) == min(len(self.d), MAXSIZE) return True def _benchmark_update(self, n): MAXSIZE=n/2 d2 = self.klass(maxsize=MAXSIZE) assert len(d2) == 0 d2.update(self.d) assert len(d2) == min(len(self.d), MAXSIZE) return True def _benchmark_insert(self, n): MAXSIZE=n/2 d2 = self.klass(maxsize=MAXSIZE) assert len(d2) == 0 for k, v, in self.d.items(): d2[k] = v assert len(d2) == min(len(self.d), MAXSIZE) return True def _benchmark_init_and_popitem(self, n): MAXSIZE=n/2 d2 = self.klass(initialdata=self.d, maxsize=MAXSIZE) assert len(d2) == min(len(self.d), MAXSIZE) for i in range(len(d2), 0, -1): assert len(d2) == i d2.popitem() return True def _benchmark_init_and_has_key_and_del(self, n): MAXSIZE=n/2 d2 = self.klass(initialdata=self.d, maxsize=MAXSIZE) assert len(d2) == min(len(self.d), MAXSIZE) for k in self.d.iterkeys(): if d2.has_key(k): del d2[k] return True def _benchmark_init_and_remove(self, n): MAXSIZE=n/2 d2 = self.klass(initialdata=self.d, maxsize=MAXSIZE) assert len(d2) == min(len(self.d), MAXSIZE) for k in self.d.iterkeys(): d2.remove(k, strictkey=False) return True def bench(self, BSIZES=(128, 250, 2048, 5000, 2**13, 2**20,)): from pyutil import benchutil funcs = ("_benchmark_insert", "_benchmark_init_and_has_key_and_del", "_benchmark_init_and_remove", "_benchmark_init_and_popitem", "_benchmark_update", "_benchmark_init",) max = 0 for func in funcs: if len(func) > max: max = len(func) for func in funcs: print(func + " " * (max + 1 - len(func))) for BSIZE in BSIZES: f = getattr(self, func) benchutil.rep_bench(f, BSIZE, self._generic_benchmarking_init, MAXREPS=self.MAXREPS, MAXTIME=self.MAXTIME) def quick_bench(): Bencher(cache.LRUCache, MAXTIME=2).bench(BSIZES=(2**7, 2**12, 2**14, 2**15, 2**16,)) Bencher(cache.LinkedListLRUCache, MAXTIME=2).bench(BSIZES=(2**7, 2**12, 2**14, 2**15,)) Bencher(cache.SmallLRUCache, MAXTIME=2).bench(BSIZES=(2**7, 2**12, 2**14, 2**15,)) def slow_bench(): Bencher(cache.LRUCache, MAXTIME=5).bench(BSIZES=[2**x for x in range(7, 21)]) Bencher(cache.LinkedListLRUCache, MAXTIME=5).bench(BSIZES=[2**x for x in range(7, 21)]) Bencher(cache.SmallLRUCache, MAXTIME=5).bench(BSIZES=[2**x for x in range(7, 17)]) MUCHADDINGSIZE=2**4 MUCHADDINGNUM = 2**4 # The following parameters are for testing for memory leakage. MIN_SLOPE = 512.0 # If it leaks less than 512.0 bytes per iteration, then it's probably just some kind of noise from the interpreter or something... SAMPLES = 2**5 # MIN_SLOPE is high because samples is low, which is because taking a statistically useful numbers of samples takes too long. # For a *good* test, turn samples up as high as you can stand (maybe 2**10) and set MIN_SLOPE to about 1.0. # For a *really* good test, add a variance measure to memutil.measure_mem_leakage(), and only consider it to be leaking if the slope is > 0.1 *and* is a "pretty good" fit for the data. # MIN_SLOPE = 1.0 # SAMPLES = 2**10 class Testy(unittest.TestCase): def _test_empty_lookup(self, d) : self.assertTrue(d.get('spam') is None) def _test_key_error(self, C) : d = C() try: d['spam'] self.fail(d) except KeyError : pass def _test_insert_and_get(self, d) : d.insert("spam", "eggs") d["spam2"] = "eggs2" self.assertTrue(d.get("spam") == "eggs", str(d)) self.assertTrue(d.get("spam2") == "eggs2") self.assertTrue(d["spam"] == "eggs") self.assertTrue(d["spam2"] == "eggs2") def _test_insert_and_remove(self, d): d.insert('spam', "eggs") self.assertTrue(d.has_key('spam')) self.assertTrue(d.get('spam') == "eggs") self.assertTrue(d['spam'] == "eggs") x = d.remove('spam') self.assertTrue(x == "eggs", "x: %r" % x) self.assertTrue(not d.has_key('spam')) d['spam'] = "eggs" self.assertTrue(d.has_key('spam')) self.assertTrue(d.get('spam') == "eggs") self.assertTrue(d['spam'] == "eggs") del d['spam'] self.assertTrue(not d.has_key('spam')) def _test_setdefault(self, d): d.setdefault('spam', "eggs") self.assertTrue(d.has_key('spam')) self.assertTrue(d.get('spam') == "eggs") self.assertTrue(d['spam'] == "eggs") x = d.remove('spam') self.assertTrue(x == "eggs", "x: %r" % x) self.assertTrue(not d.has_key('spam')) def _test_extracted_bound_method(self, d): insmeth = d.insert insmeth('spammy', "eggsy") self.assertTrue(d.get('spammy') == "eggsy") def _test_extracted_unbound_method(self, d): insumeth = d.__class__.insert insumeth(d, 'spammy', "eggsy") self.assertTrue(d.get('spammy') == "eggsy") def _test_unbound_method(self, C, d): umeth = C.insert umeth(d, 'spammy', "eggsy") self.assertTrue(d.get('spammy') == "eggsy") def _test_clear(self, d): d[11] = 11 d._assert_invariants() self.assertTrue(len(d) == 1) d.clear() d._assert_invariants() self.assertTrue(len(d) == 0) def _test_update(self, d): self.assertTrue(d._assert_invariants()) d['b'] = 99 self.assertTrue(d._assert_invariants()) d2={ 'a': 0, 'b': 1, 'c': 2,} d.update(d2) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('a') == 0, "d.get('a'): %s" % d.get('a')) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('b') == 1, "d.get('b'): %s" % d.get('b')) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('c') == 2) self.assertTrue(d._assert_invariants()) def _test_popitem(self, C): c = C({"a": 1}) res = c.popitem() _assert(res == ("a", 1,), C, c, res) self.assertTrue(res == ("a", 1,)) def _test_iterate_items(self, C): c = C({"a": 1}) i = c.iteritems() x = i.next() self.assertTrue(x == ("a", 1,)) try: i.next() self.fail() # Should have gotten StopIteration exception except StopIteration: pass def _test_iterate_keys(self, C): c = C({"a": 1}) i = c.iterkeys() x = i.next() self.assertTrue(x == "a") try: i.next() self.fail() # Should have gotten StopIteration exception except StopIteration: pass def _test_iterate_values(self, C): c = C({"a": 1}) i = c.itervalues() x = i.next() self.assertEqual(x, 1) try: i.next() self.fail() # Should have gotten StopIteration exception except StopIteration: pass def _test_LRU_much_adding_some_removing(self, C): c = C(maxsize=MUCHADDINGSIZE) for i in range(MUCHADDINGNUM): c[i] = i if (i % 400) == 0: k = random.choice(c.keys()) del c[k] for i in range(MUCHADDINGSIZE): c[i] = i self.assertTrue(len(c) == MUCHADDINGSIZE) def _test_LRU_1(self, C): c = C(maxsize=10) c[11] = 11 c._assert_invariants() c[11] = 11 c._assert_invariants() c[11] = 1001 c._assert_invariants() c[11] = 11 c._assert_invariants() c[11] = 1001 c._assert_invariants() c[11] = 1001 c._assert_invariants() c[11] = 1001 c._assert_invariants() def _test_LRU_2(self, C): c = C(maxsize=10) c[11] = 11 c._assert_invariants() del c[11] c._assert_invariants() c[11] = 11 c._assert_invariants() c[11] = 11 c._assert_invariants() def _test_LRU_3(self, C): c = C(maxsize=10) c[11] = 11 c._assert_invariants() c[11] = 12 c._assert_invariants() c[11] = 13 c._assert_invariants() del c[11] c._assert_invariants() c[11] = 14 c._assert_invariants() c[11] = 15 c._assert_invariants() c[11] = 16 c._assert_invariants() def _test_LRU_full(self, C): c = C(maxsize=10) c._assert_invariants() for i in range(11): c._assert_invariants() c[i] = i c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(10 in c.values(), c.values()) self.assertTrue(0 not in c.values()) del c[1] c._assert_invariants() self.assertTrue(1 not in c.values()) self.assertTrue(len(c) == 9) c[11] = 11 c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(1 not in c.values()) self.assertTrue(11 in c.values()) del c[11] c._assert_invariants() c[11] = 11 c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(1 not in c.values()) self.assertTrue(11 in c.values()) c[11] = 11 c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(1 not in c.values()) self.assertTrue(11 in c.values()) for i in range(200): c[i] = i c._assert_invariants() self.assertTrue(199 in c.values()) self.assertTrue(190 in c.values()) def _test_LRU_has_key(self, C): c = C(maxsize=10) c._assert_invariants() for i in range(11): c._assert_invariants() c[i] = i c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(10 in c.values()) self.assertTrue(0 not in c.values()) # c.has_key(1) # this touches `1' and makes it fresher so that it will live and `2' will die next time we overfill. c[1] = 1 # this touches `1' and makes it fresher so that it will live and `2' will die next time we overfill. c._assert_invariants() c[99] = 99 c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(1 in c.values(), "C: %s, c.values(): %s" % (hr(C), hr(c.values(),),)) self.assertTrue(not 2 in c.values()) self.assertTrue(99 in c.values()) def _test_LRU_not_overfull_on_idempotent_add(self, C): c = C(maxsize=10) for i in range(11): c[i] = i c[1] = "spam" # Now 1 is the freshest, so 2 is the next one that would be removed *if* we went over limit. c[3] = "eggs" self.assertTrue(c.has_key(2)) self.assertTrue(len(c) == 10) c._assert_invariants() def _test_LRU_overflow_on_update(self, C): d = C(maxsize=10) self.assertTrue(d._assert_invariants()) d2 = {} for i in range(12): d2[i] = i d.update(d2) self.assertTrue(d._assert_invariants()) self.assertTrue(len(d) == 10) def _test_LRU_overflow_on_init(self, C): d2 = {} for i in range(12): d2[i] = i d = C(d2, maxsize=10) self.assertTrue(d._assert_invariants()) self.assertTrue(len(d) == 10) def _test_em(self): for klass in (cache.LRUCache, cache.SmallLRUCache,): for testfunc in (self._test_empty_lookup, self._test_insert_and_get, self._test_insert_and_remove, self._test_extracted_bound_method, self._test_extracted_unbound_method, self._test_clear, self._test_update, self._test_setdefault,): testfunc(klass()) for testfunc in (self._test_popitem, self._test_iterate_items, self._test_iterate_keys, self._test_iterate_values, self._test_key_error, ): testfunc(klass) self._test_unbound_method(klass, klass()) for klass in (cache.LRUCache, cache.SmallLRUCache,): for testfunc in (self._test_LRU_1, self._test_LRU_2, self._test_LRU_3, self._test_LRU_full, self._test_LRU_has_key, self._test_LRU_not_overfull_on_idempotent_add, self._test_LRU_overflow_on_update, self._test_LRU_overflow_on_init,): testfunc(klass) def test_em(self): self._test_em() def _mem_test_LRU_much_adding_some_removing(self): for klass in (cache.LRUCache, cache.SmallLRUCache,): return self._test_LRU_much_adding_some_removing(klass) def test_mem_leakage(self): try: self._test_mem_leakage() except memutil.NotSupportedException: print("Skipping memory leak test since measurement of current mem usage isn't implemented on this platform.") pass del test_mem_leakage # This test takes too long. def _test_mem_leakage(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of memory state. memutil.measure_mem_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) slope = memutil.measure_mem_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks memory at a rate of approximately %s system bytes per invocation" % (self.test_em, "%0.3f" % slope,)) def test_mem_leakage_much_adding_some_removing(self): try: self._test_mem_leakage_much_adding_some_removing() except memutil.NotSupportedException: print("Skipping memory leak test since measurement of current mem usage isn't implemented on this platform.") pass del test_mem_leakage_much_adding_some_removing # This test takes too long. def _test_mem_leakage_much_adding_some_removing(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of memory state. memutil.measure_mem_leakage(self._mem_test_LRU_much_adding_some_removing, SAMPLES, iterspersample=2**0) slope = memutil.measure_mem_leakage(self._mem_test_LRU_much_adding_some_removing, SAMPLES, iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks memory at a rate of approximately %s system bytes per invocation" % (self._mem_test_LRU_much_adding_some_removing, "%0.3f" % slope,)) def test_obj_leakage(self): self._test_obj_leakage() del test_obj_leakage # This test takes too long. def _test_obj_leakage(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of objects state. memutil.measure_obj_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) slope = memutil.measure_obj_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks objects at a rate of approximately %s system bytes per invocation" % (self.test_em, "%0.3f" % slope,)) def test_obj_leakage_much_adding_some_removing(self): self._test_obj_leakage_much_adding_some_removing() del test_obj_leakage_much_adding_some_removing # This test takes too long. def _test_obj_leakage_much_adding_some_removing(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of objects state. memutil.measure_obj_leakage(self._mem_test_LRU_much_adding_some_removing, SAMPLES, iterspersample=2**0) slope = memutil.measure_obj_leakage(self._mem_test_LRU_much_adding_some_removing, SAMPLES, iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks objects at a rate of approximately %s system bytes per invocation" % (self._mem_test_LRU_much_adding_some_removing, "%0.3f" % slope,)) pyutil-3.3.2/pyutil/test/out_of_shape/test_odict.py000066400000000000000000000404261437014040400225430ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. from __future__ import print_function import random, unittest from pyutil.humanreadable import hr from pyutil import memutil from pyutil import odict class Bencher: def __init__(self, klass, MAXREPS=2**8, MAXTIME=5): print(klass) self.klass = klass self.MAXREPS = MAXREPS self.MAXTIME = MAXTIME self.d = {} self.lrun = None def _generic_benchmarking_init(self, n): self.d.clear() self.lrun = self.klass() for i in range(n): self.d[i] = i self.lrun[n+i] = n+i def _benchmark_init(self, n): d2 = self.klass(initialdata=self.d) assert len(d2) == len(self.d) return True def _benchmark_update(self, n): d2 = self.klass() assert len(d2) == 0 d2.update(self.d) assert len(d2) == len(self.d) return True def _benchmark_insert(self, n): d2 = self.klass() assert len(d2) == 0 for k, v, in self.d.iteritems(): d2[k] = v assert len(d2) == len(self.d) return True def _benchmark_init_and_popitem(self, n): d2 = self.klass(initialdata=self.d) assert len(d2) == len(self.d) for i in range(len(d2), 0, -1): assert len(d2) == i d2.popitem() return True def _benchmark_init_and_has_key_and_del(self, n): d2 = self.klass(initialdata=self.d) assert len(d2) == len(self.d) for k in self.d.iterkeys(): if d2.has_key(k): del d2[k] return True def _benchmark_init_and_remove(self, n): d2 = self.klass(initialdata=self.d) assert len(d2) == len(self.d) for k in self.d.iterkeys(): d2.remove(k, strictkey=False) return True def bench(self, BSIZES=(128, 250, 2048, 5000, 2**13, 2**20,)): from pyutil import benchutil funcs = ("_benchmark_insert", "_benchmark_init_and_has_key_and_del", "_benchmark_init_and_remove", "_benchmark_init_and_popitem", "_benchmark_update", "_benchmark_init",) max = 0 for func in funcs: if len(func) > max: max = len(func) for func in funcs: print(func + " " * (max + 1 - len(func))) for BSIZE in BSIZES: f = getattr(self, func) benchutil.rep_bench(f, BSIZE, self._generic_benchmarking_init, MAXREPS=self.MAXREPS, MAXTIME=self.MAXTIME) def quick_bench(): Bencher(odict.LRUCache, MAXTIME=2).bench(BSIZES=(2**7, 2**12, 2**14, 2**15, 2**16,)) Bencher(odict.LinkedListLRUCache, MAXTIME=2).bench(BSIZES=(2**7, 2**12, 2**14, 2**15,)) Bencher(odict.SmallLRUCache, MAXTIME=2).bench(BSIZES=(2**7, 2**12, 2**14, 2**15,)) def slow_bench(): Bencher(odict.LRUCache, MAXTIME=5).bench(BSIZES=[2**x for x in range(7, 21)]) Bencher(odict.LinkedListLRUCache, MAXTIME=5).bench(BSIZES=[2**x for x in range(7, 21)]) Bencher(odict.SmallLRUCache, MAXTIME=5).bench(BSIZES=[2**x for x in range(7, 17)]) MUCHADDINGSIZE=2**4 # The following parameters are for testing for memory leakage. MIN_SLOPE = 512.0 # If it leaks less than 512.0 bytes per iteration, then it's probably just some kind of noise from the interpreter or something... SAMPLES = 2**5 # MIN_SLOPE is high because samples is low, which is because taking a statistically useful numbers of samples takes too long. # For a *good* test, turn samples up as high as you can stand (maybe 2**10) and set MIN_SLOPE to about 1.0. # For a *really* good test, add a variance measure to memutil.measure_mem_leakage(), and only consider it to be leaking if the slope is > 0.1 *and* is a "pretty good" fit for the data. # MIN_SLOPE = 1.0 # SAMPLES = 2**10 class Testy(unittest.TestCase): def _test_empty_lookup(self, d) : self.assertTrue(d.get('spam') is None) def _test_key_error(self, C) : d = C() try: d['spam'] self.fail(d) except KeyError : pass def _test_insert_and_get_and_items(self, d) : d.insert("spam", "eggs") d["spam2"] = "eggs2" self.assertTrue(d.get("spam") == "eggs", str(d)) self.assertTrue(d.get("spam2") == "eggs2") self.assertTrue(d["spam"] == "eggs") self.assertTrue(d["spam2"] == "eggs2") self.assertEqual(list(d.items()), [("spam", "eggs"), ("spam2", "eggs2")], d) def _test_move_to_most_recent(self, d) : d.insert("spam", "eggs") d["spam2"] = "eggs2" self.assertTrue(d.get("spam") == "eggs", str(d)) self.assertTrue(d.get("spam2") == "eggs2") self.assertTrue(d["spam"] == "eggs") self.assertTrue(d["spam2"] == "eggs2") self.assertEqual(d.items(), [("spam", "eggs"), ("spam2", "eggs2")]) d.move_to_most_recent("spam") self.assertEqual(list(d.items()), [("spam2", "eggs2"), ("spam", "eggs")]) def _test_insert_and_remove(self, d): d.insert('spam', "eggs") self.assertTrue(d.has_key('spam')) self.assertTrue(d.get('spam') == "eggs") self.assertTrue(d['spam'] == "eggs") self.assertEqual(list(d.items()), [("spam", "eggs")]) x = d.remove('spam') self.assertTrue(x == "eggs", "x: %r" % x) self.assertTrue(not d.has_key('spam')) self.assertEqual(list(d.items()), []) d['spam'] = "eggsy" self.assertTrue(d.has_key('spam')) self.assertTrue(d.get('spam') == "eggsy") self.assertTrue(d['spam'] == "eggsy") self.assertEqual(list(d.items()), [("spam", "eggsy")]) del d['spam'] self.assertTrue(not d.has_key('spam')) self.assertEqual(list(d.items()), []) def _test_setdefault(self, d): d.setdefault('spam', "eggs") self.assertTrue(d.has_key('spam')) self.assertTrue(d.get('spam') == "eggs") self.assertTrue(d['spam'] == "eggs") self.assertEqual(list(d.items()), [("spam", "eggs")]) x = d.remove('spam') self.assertTrue(x == "eggs", "x: %r" % x) self.assertTrue(not d.has_key('spam')) self.assertEqual(list(d.items()), []) def _test_extracted_bound_method(self, d): insmeth = d.insert insmeth('spammy', "eggsy") self.assertTrue(d.get('spammy') == "eggsy") def _test_extracted_unbound_method(self, d): insumeth = d.__class__.insert insumeth(d, 'spammy', "eggsy") self.assertTrue(d.get('spammy') == "eggsy") def _test_unbound_method(self, C, d): umeth = C.insert umeth(d, 'spammy', "eggsy") self.assertTrue(d.get('spammy') == "eggsy") def _test_clear(self, d): d[11] = 11 d._assert_invariants() self.assertTrue(len(d) == 1) d.clear() d._assert_invariants() self.assertTrue(len(d) == 0) self.assertEqual(list(d.items()), []) def _test_update_from_dict(self, d): self.assertTrue(d._assert_invariants()) d['b'] = 99 self.assertTrue(d._assert_invariants()) d2={ 'a': 0, 'b': 1, 'c': 2,} d.update(d2) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('a') == 0, "d.get('a'): %s" % d.get('a')) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('b') == 1, "d.get('b'): %s" % d.get('b')) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('c') == 2) self.assertTrue(d._assert_invariants()) def _test_update_from_odict(self, d): self.assertTrue(d._assert_invariants()) d['b'] = 99 self.assertTrue(d._assert_invariants()) d2 = odict.OrderedDict() d2['a'] = 0 d2['b'] = 1 d2['c'] = 2 d.update(d2) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('a') == 0, "d.get('a'): %s" % d.get('a')) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('b') == 1, "d.get('b'): %s" % d.get('b')) self.assertTrue(d._assert_invariants()) self.assertTrue(d.get('c') == 2) self.assertTrue(d._assert_invariants()) self.assertEqual(list(d.items()), [("b", 1), ("a", 0), ("c", 2)]) def _test_popitem(self, C): c = C({"a": 1}) res = c.popitem() self.assertEqual(res, ("a", 1,)) c["a"] = 1 c["b"] = 2 res = c.popitem() self.assertEqual(res, ("b", 2,)) def _test_pop(self, C): c = C({"a": 1}) res = c.pop() self.assertEqual(res, "a") c["a"] = 1 c["b"] = 2 res = c.pop() self.assertEqual(res, "b") def _test_iterate_items(self, C): c = C({"a": 1}) c["b"] = 2 i = c.iteritems() x = i.next() self.assertEqual(x, ("a", 1,)) x = i.next() self.assertEqual(x, ("b", 2,)) try: i.next() self.fail() # Should have gotten StopIteration exception except StopIteration: pass def _test_iterate_keys(self, C): c = C({"a": 1}) c["b"] = 2 i = c.iterkeys() x = i.next() self.assertEqual(x, "a") x = i.next() self.assertEqual(x, "b") try: i.next() self.fail() # Should have gotten StopIteration exception except StopIteration: pass def _test_iterate_values(self, C): c = C({"a": 1}) c["b"] = 2 i = c.itervalues() x = i.next() self.assertTrue(x == 1) x = i.next() self.assertTrue(x == 2) try: i.next() self.fail() # Should have gotten StopIteration exception except StopIteration: pass def _test_much_adding_some_removing(self, C): c = C() for i in range(MUCHADDINGSIZE): c[i] = i if (i % 4) == 0: k = random.choice(c.keys()) del c[k] for i in range(MUCHADDINGSIZE): c[i] = i self.assertEqual(len(c), MUCHADDINGSIZE) def _test_1(self, C): c = C() c[11] = 11 c._assert_invariants() c[11] = 11 c._assert_invariants() c[11] = 1001 c._assert_invariants() c[11] = 11 c._assert_invariants() c[11] = 1001 c._assert_invariants() c[11] = 1001 c._assert_invariants() c[11] = 1001 c._assert_invariants() def _test_2(self, C): c = C() c[11] = 11 c._assert_invariants() del c[11] c._assert_invariants() c[11] = 11 c._assert_invariants() c[11] = 11 c._assert_invariants() def _test_3(self, C): c = C() c[11] = 11 c._assert_invariants() c[11] = 12 c._assert_invariants() c[11] = 13 c._assert_invariants() del c[11] c._assert_invariants() c[11] = 14 c._assert_invariants() c[11] = 15 c._assert_invariants() c[11] = 16 c._assert_invariants() def _test_has_key(self, C): c = C() c._assert_invariants() for i in range(11): c._assert_invariants() c[i] = i c._assert_invariants() del c[0] self.assertTrue(len(c) == 10) self.assertTrue(10 in c.values()) self.assertTrue(0 not in c.values()) c.has_key(1) # this touches `1' but does not make it fresher so that it will get popped next time we pop. c[1] = 1 # this touches `1' but does not make it fresher so that it will get popped. c._assert_invariants() x = c.pop() self.assertEqual(x, 10) c[99] = 99 c._assert_invariants() self.assertTrue(len(c) == 10) self.assertTrue(1 in c.values(), "C: %s, c.values(): %s" % (hr(C), hr(c.values(),),)) self.assertTrue(2 in c.values(), "C: %s, c.values(): %s" % (hr(C), hr(c.values(),),)) self.assertFalse(10 in c.values(), "C: %s, c.values(): %s" % (hr(C), hr(c.values(),),)) self.assertTrue(99 in c.values()) def _test_em(self): for klass in (odict.OrderedDict,): for testfunc in (self._test_empty_lookup, self._test_insert_and_get_and_items, self._test_insert_and_remove, self._test_extracted_bound_method, self._test_extracted_unbound_method, self._test_clear, self._test_update_from_dict, self._test_update_from_odict, self._test_setdefault,): testfunc(klass()) for testfunc in (self._test_pop, self._test_popitem, self._test_iterate_items, self._test_iterate_keys, self._test_iterate_values, self._test_key_error, ): testfunc(klass) self._test_unbound_method(klass, klass()) for klass in (odict.OrderedDict,): for testfunc in (self._test_1, self._test_2, self._test_3, self._test_has_key,): testfunc(klass) def test_em(self): self._test_em() def _mem_test_much_adding_some_removing(self): for klass in (odict.LRUCache, odict.SmallLRUCache,): return self._test_much_adding_some_removing(klass) def test_mem_leakage(self): try: self._test_mem_leakage() except memutil.NotSupportedException: print("Skipping memory leak test since measurement of current mem usage isn't implemented on this platform.") pass del test_mem_leakage # This test takes too long. def _test_mem_leakage(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of memory state. memutil.measure_mem_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) slope = memutil.measure_mem_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks memory at a rate of approximately %s system bytes per invocation" % (self.test_em, "%0.3f" % slope,)) def test_mem_leakage_much_adding_some_removing(self): try: self._test_mem_leakage_much_adding_some_removing() except memutil.NotSupportedException: print("Skipping memory leak test since measurement of current mem usage isn't implemented on this platform.") pass del test_mem_leakage_much_adding_some_removing # This test takes too long. def _test_mem_leakage_much_adding_some_removing(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of memory state. memutil.measure_mem_leakage(self._mem_test_much_adding_some_removing, SAMPLES, iterspersample=2**0) slope = memutil.measure_mem_leakage(self._mem_test_much_adding_some_removing, SAMPLES, iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks memory at a rate of approximately %s system bytes per invocation" % (self._mem_test_much_adding_some_removing, "%0.3f" % slope,)) def test_obj_leakage(self): self._test_obj_leakage() del test_obj_leakage # This test takes too long. def _test_obj_leakage(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of objects state. memutil.measure_obj_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) slope = memutil.measure_obj_leakage(self.test_em, max(2**3, SAMPLES/2**3), iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks objects at a rate of approximately %s system bytes per invocation" % (self.test_em, "%0.3f" % slope,)) def test_obj_leakage_much_adding_some_removing(self): self._test_obj_leakage_much_adding_some_removing() del test_obj_leakage_much_adding_some_removing # This test takes too long. def _test_obj_leakage_much_adding_some_removing(self): # measure one and throw it away, in order to reach a "steady state" in terms of initialization of objects state. memutil.measure_obj_leakage(self._mem_test_much_adding_some_removing, SAMPLES, iterspersample=2**0) slope = memutil.measure_obj_leakage(self._mem_test_much_adding_some_removing, SAMPLES, iterspersample=2**0) self.assertTrue(slope <= MIN_SLOPE, "%s leaks objects at a rate of approximately %s system bytes per invocation" % (self._mem_test_much_adding_some_removing, "%0.3f" % slope,)) pyutil-3.3.2/pyutil/test/out_of_shape/test_strutil.py000066400000000000000000000033271437014040400231460ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest from pyutil.assertutil import _assert from pyutil import strutil class Teststrutil(unittest.TestCase): def test_short_input(self): self.assertTrue(strutil.pop_trailing_newlines("\r\n") == "") self.assertTrue(strutil.pop_trailing_newlines("\r") == "") self.assertTrue(strutil.pop_trailing_newlines("x\r\n") == "x") self.assertTrue(strutil.pop_trailing_newlines("x\r") == "x") def test_split(self): _assert(strutil.split_on_newlines("x\r\ny") == ["x", "y",], strutil.split_on_newlines("x\r\ny")) _assert(strutil.split_on_newlines("x\r\ny\r\n") == ["x", "y", '',], strutil.split_on_newlines("x\r\ny\r\n")) _assert(strutil.split_on_newlines("x\n\ny\n\n") == ["x", '', "y", '', '',], strutil.split_on_newlines("x\n\ny\n\n")) def test_commonprefix(self): _assert(strutil.commonprefix(["foo","foobarooo", "foosplat",]) == 'foo', strutil.commonprefix(["foo","foobarooo", "foosplat",])) _assert(strutil.commonprefix(["foo","afoobarooo", "foosplat",]) == '', strutil.commonprefix(["foo","afoobarooo", "foosplat",])) def test_commonsuffix(self): _assert(strutil.commonsuffix(["foo","foobarooo", "foosplat",]) == '', strutil.commonsuffix(["foo","foobarooo", "foosplat",])) _assert(strutil.commonsuffix(["foo","foobarooo", "foosplato",]) == 'o', strutil.commonsuffix(["foo","foobarooo", "foosplato",])) _assert(strutil.commonsuffix(["foo","foobarooofoo", "foosplatofoo",]) == 'foo', strutil.commonsuffix(["foo","foobarooofoo", "foosplatofoo",])) pyutil-3.3.2/pyutil/test/out_of_shape/test_zlibutil.py000066400000000000000000000065471437014040400233050ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import unittest from pyutil import randutil from pyutil import zlibutil class Accumulator: def __init__(self): self.buf = b'' def write(self, str): self.buf += str def make_decomp(realdecomp): def decomp(str, maxlen, maxmem): d = Accumulator() realdecomp(str, d, maxlen, maxmem) return d.buf return decomp def genrandstr(strlen): return randutil.insecurerandstr(strlen) def genbombstr(strlen): return b'0' * strlen MAXMEM=65*2**20 class ZlibTestCase(unittest.TestCase): def _help_test(self, genstring, decomp, strlen): s = genstring(strlen) cs = zlibutil.zlib.compress(s) s2 = decomp(cs, maxlen=strlen, maxmem=strlen*2**3 + zlibutil.MINMAXMEM) self.assertTrue(s == s2) s2 = decomp(cs, maxlen=strlen, maxmem=strlen*2**6 + zlibutil.MINMAXMEM) self.assertTrue(s == s2) self.assertRaises(zlibutil.TooBigError, decomp, cs, maxlen=strlen-1, maxmem=strlen*2**3 + zlibutil.MINMAXMEM) def _help_test_inplace_minmaxmem(self, genstring, decomp, strlen): s = genstring(strlen) cs = zlibutil.zlib.compress(s) s2 = decomp(cs, maxlen=strlen, maxmem=zlibutil.MINMAXMEM) self.assertTrue(s == s2) self.assertRaises(zlibutil.TooBigError, decomp, cs, maxlen=strlen-1, maxmem=zlibutil.MINMAXMEM) def _help_test_inplace(self, genstring, decomp, strlen): # ### XXX self.assertRaises(UnsafeDecompressError, decomp, zlib.compress(genstring(strlen)), maxlen=strlen, maxmem=strlen-1) s = genstring(strlen) cs = zlibutil.zlib.compress(s) s2 = decomp(cs, maxlen=strlen, maxmem=max(strlen*2**3, zlibutil.MINMAXMEM)) self.assertTrue(s == s2) s2 = decomp(cs, maxlen=strlen, maxmem=max(strlen*2**6, zlibutil.MINMAXMEM)) self.assertTrue(s == s2) s2 = decomp(cs, maxlen=strlen, maxmem=max(strlen-1, zlibutil.MINMAXMEM)) self.assertTrue(s == s2) s2 = decomp(cs, maxlen=strlen, maxmem=max(strlen/2, zlibutil.MINMAXMEM)) self.assertTrue(s == s2) self.assertRaises(zlibutil.TooBigError, decomp, cs, maxlen=strlen-1, maxmem=max(strlen*2**3, zlibutil.MINMAXMEM)) def testem(self): # for strlen in [2**1, 2**2, 2**10, 2**14, 2**21]: # a *real* test ought to include 2**21, which exercises different cases re: maxmem. But it takes too long. for strlen in [2, 3, 4, 99,]: # print "strlen: %s\n" % (strlen,) for decomp in [zlibutil.decompress, make_decomp(zlibutil.decompress_to_fileobj), make_decomp(zlibutil.decompress_to_spool),]: # print "decomp: %s\n" % (decomp,) for genstring in [genrandstr, genbombstr,]: # print "genstring: %s\n" % (genstring,) self._help_test(genstring, decomp, strlen) for decomp in [make_decomp(zlibutil.decompress_to_spool),]: # print "decomp: %s\n" % (decomp,) for genstring in [genrandstr, genbombstr,]: # print "genstring: %s\n" % (genstring,) self._help_test_inplace(genstring, decomp, strlen) self._help_test_inplace_minmaxmem(genstring, decomp, strlen) pyutil-3.3.2/pyutil/testutil.py000066400000000000000000000117241437014040400166240ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- from __future__ import print_function import os, signal, time from twisted.internet import defer, reactor from twisted.trial import unittest from . import repeatable_random repeatable_random # http://divmod.org/trac/ticket/1499 class SignalMixin: # This class is necessary for any code which wants to use Processes # outside the usual reactor.run() environment. It is copied from # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses # something rather different. sigchldHandler = None def setUp(self): # make sure SIGCHLD handler is installed, as it should be on # reactor.run(). problem is reactor may not have been run when this # test runs. if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): self.sigchldHandler = signal.signal(signal.SIGCHLD, reactor._handleSigchld) def tearDown(self): if self.sigchldHandler: signal.signal(signal.SIGCHLD, self.sigchldHandler) class PollMixin: def poll(self, check_f, pollinterval=0.01): # Return a Deferred, then call check_f periodically until it returns # True, at which point the Deferred will fire.. If check_f raises an # exception, the Deferred will errback. d = defer.maybeDeferred(self._poll, None, check_f, pollinterval) return d def _poll(self, res, check_f, pollinterval): if check_f(): return True d = defer.Deferred() d.addCallback(self._poll, check_f, pollinterval) reactor.callLater(pollinterval, d.callback, None) return d class TestMixin(SignalMixin): def setUp(self, repeatable=False): """ @param repeatable: install the repeatable_randomness hacks to attempt to without access to real randomness and real time.time from the code under test """ self.repeatable = repeatable if self.repeatable: import repeatable_random repeatable_random.force_repeatability() if hasattr(time, 'realtime'): self.teststarttime = time.realtime() else: self.teststarttime = time.time() def tearDown(self): if self.repeatable: repeatable_random.restore_non_repeatability() self.clean_pending(required_to_quiesce=True) def clean_pending(self, dummy=None, required_to_quiesce=True): """ This handy method cleans all pending tasks from the reactor. When writing a unit test, consider the following question: Is the code that you are testing required to release control once it has done its job, so that it is impossible for it to later come around (with a delayed reactor task) and do anything further? If so, then trial will usefully test that for you -- if the code under test leaves any pending tasks on the reactor then trial will fail it. On the other hand, some code is *not* required to release control -- some code is allowed to continuously maintain control by rescheduling reactor tasks in order to do ongoing work. Trial will incorrectly require that code to clean up all its tasks from the reactor. Most people think that such code should be amended to have an optional "shutdown" operation that releases all control, but on the contrary it is good design for some code to *not* have a shutdown operation, but instead to have a "crash-only" design in which it recovers from crash on startup. If the code under test is of the "long-running" kind, which is *not* required to shutdown cleanly in order to pass tests, then you can simply call testutil.clean_pending() at the end of the unit test, and trial will be satisfied. """ pending = reactor.getDelayedCalls() active = bool(pending) for p in pending: if p.active(): p.cancel() else: print("WEIRDNESS! pending timed call not active!") if required_to_quiesce and active: self.fail("Reactor was still active when it was required to be quiescent.") try: import win32file import win32con def w_make_readonly(path): win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_READONLY) def w_make_accessible(path): win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) # http://divmod.org/trac/ticket/1499 make_readonly = w_make_readonly make_accessible = w_make_accessible except ImportError: import stat def make_readonly(path): os.chmod(path, stat.S_IREAD) os.chmod(os.path.dirname(path), stat.S_IREAD) def make_accessible(path): os.chmod(os.path.dirname(path), stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) pyutil-3.3.2/pyutil/time_format.py000066400000000000000000000045021437014040400172510ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # ISO-8601: # http://www.cl.cam.ac.uk/~mgk25/iso-time.html import calendar, datetime, re, time def iso_utc_date(now=None, t=time.time): if now is None: now = t() return datetime.datetime.utcfromtimestamp(now).isoformat()[:10] def iso_utc(now=None, sep=' ', t=time.time, suffix='Z'): if now is None: now = t() return datetime.datetime.utcfromtimestamp(now).isoformat(sep)+suffix def iso_local(now=None, sep=' ', t=time.time): if now is None: now = t() return datetime.datetime.fromtimestamp(now).isoformat(sep) def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})[T_ ](?P\d{2}):(?P\d{2}):(?P\d{2})(?P\.\d+)?Z?")): """ The inverse of iso_utc(). Real ISO-8601 is "2003-01-08T06:30:59Z". We also accept "2003-01-08 06:30:59Z" as suggested by RFC 3339. We also accept "2003-01-08_06:30:59Z". We also accept the trailing 'Z' to be omitted. """ m = _conversion_re.match(isotime) if not m: raise ValueError(isotime, "not a complete ISO8601 timestamp") year, month, day = int(m.group('year')), int(m.group('month')), int(m.group('day')) hour, minute, second = int(m.group('hour')), int(m.group('minute')), int(m.group('second')) subsecstr = m.group('subsecond') if subsecstr: subsecfloat = float(subsecstr) else: subsecfloat = 0 return calendar.timegm( (year, month, day, hour, minute, second, 0, 1, 0) ) + subsecfloat def parse_duration(s): orig = s unit = None DAY = 24*60*60 MONTH = 31*DAY YEAR = 365*DAY if s.endswith("s"): s = s[:-1] if s.endswith("day"): unit = DAY s = s[:-len("day")] elif s.endswith("month"): unit = MONTH s = s[:-len("month")] elif s.endswith("mo"): unit = MONTH s = s[:-len("mo")] elif s.endswith("year"): unit = YEAR s = s[:-len("YEAR")] else: raise ValueError("no unit (like day, month, or year) in '%s'" % orig) s = s.strip() return int(s) * unit def parse_date(s): # return seconds-since-epoch for the UTC midnight that starts the given # day return int(iso_utc_time_to_seconds(s + "T00:00:00")) pyutil-3.3.2/pyutil/twistedutil.py000066400000000000000000000017301437014040400173240ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import warnings # from the Twisted library from twisted.internet import reactor # from the pyutil library from .weakutil import WeakMethod def callLater_weakly(delay, func, *args, **kwargs): """ Call func later, but if func is a bound method then make the reference it holds to object be a weak reference. Therefore, if this scheduled event is a bound method and it is the only thing keeping the object from being garbage collected, the object will be garbage collected and the event will be cancelled. """ warnings.warn("deprecated", DeprecationWarning) def cleanup(weakmeth, thedeadweakref): if weakmeth.callId.active(): weakmeth.callId.cancel() weakmeth = WeakMethod(func, callback=cleanup) weakmeth.callId = reactor.callLater(delay, weakmeth, *args, **kwargs) return weakmeth pyutil-3.3.2/pyutil/verlib.py000066400000000000000000000277701437014040400162420ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- """ "Rational" version definition and parsing for DistutilsVersionFight discussion at PyCon 2009. """ import re try: unicode = unicode except NameError: basestring = (str, bytes) class IrrationalVersionError(Exception): """This is an irrational version.""" pass class HugeMajorVersionNumError(IrrationalVersionError): """An irrational version because the major version number is huge (often because a year or date was used). See `error_on_huge_major_num` option in `NormalizedVersion` for details. This guard can be disabled by setting that option False. """ pass class PreconditionViolationException(Exception): pass # A marker used in the second and third parts of the `parts` tuple, for # versions that don't have those segments, to sort properly. An example # of versions in sort order ('highest' last): # 1.0b1 ((1,0), ('b',1), ('f',)) # 1.0.dev345 ((1,0), ('f',), ('dev', 345)) # 1.0 ((1,0), ('f',), ('f',)) # 1.0.post256.dev345 ((1,0), ('f',), ('f', 'post', 256, 'dev', 345)) # 1.0.post345 ((1,0), ('f',), ('f', 'post', 345, 'f')) # ^ ^ ^ # 'b' < 'f' ---------------------/ | | # | | # 'dev' < 'f' < 'post' -------------------/ | # | # 'dev' < 'f' ----------------------------------------------/ # Other letters would do, but 'f' for 'final' is kind of nice. FINAL_MARKER = ('f',) VERSION_RE = re.compile(r''' ^ (?P\d+\.\d+) # minimum 'N.N' (?P(?:\.\d+)*) # any number of extra '.N' segments (?: (?P[abc]|rc) # 'a'=alpha, 'b'=beta, 'c'=release candidate # 'rc'= alias for release candidate (?P\d+(?:\.\d+)*) )? (?P(\.post(?P\d+)|-r(?P\d+))?(\.dev(?P\d+))?)? $''', re.VERBOSE) class NormalizedVersion(object): """A rational version. Good: 1.2 # equivalent to "1.2.0" 1.2.0 1.2a1 1.2.3a2 1.2.3b1 1.2.3c1 1.2.3.4 TODO: fill this out Bad: 1 # mininum two numbers 1.2a # release level must have a release serial 1.2.3b """ def __init__(self, s, error_on_huge_major_num=True): """Create a NormalizedVersion instance from a version string. @param s {str} The version string. @param error_on_huge_major_num {bool} Whether to consider an apparent use of a year or full date as the major version number an error. Default True. One of the observed patterns on PyPI before the introduction of `NormalizedVersion` was version numbers like this: 2009.01.03 20040603 2005.01 This guard is here to strongly encourage the package author to use an alternate version, because a release deployed into PyPI and, e.g. downstream Linux package managers, will forever remove the possibility of using a version number like "1.0" (i.e. where the major number is less than that huge major number). """ self._parse(s, error_on_huge_major_num) @classmethod def from_parts(cls, version, prerelease=FINAL_MARKER, devpost=FINAL_MARKER): return cls(cls.parts_to_str((version, prerelease, devpost))) def _parse(self, s, error_on_huge_major_num=True): """Parses a string version into parts.""" if not isinstance(s, basestring): raise PreconditionViolationException("s is required to be a string: %s :: %s" % (s, type(s))) match = VERSION_RE.search(s) if not match: raise IrrationalVersionError(s) groups = match.groupdict() parts = [] # main version block = self._parse_numdots(groups['version'], s, False, 2) extraversion = groups.get('extraversion') if extraversion not in ('', None): block += self._parse_numdots(extraversion[1:], s) parts.append(tuple(block)) # prerelease prerel = groups.get('prerel') if prerel is not None: block = [prerel] block += self._parse_numdots(groups.get('prerelversion'), s, pad_zeros_length=1) parts.append(tuple(block)) else: parts.append(FINAL_MARKER) # postdev if groups.get('postdev'): post = groups.get('post') or groups.get('oldpost') dev = groups.get('dev') postdev = [] if post is not None: postdev.extend([FINAL_MARKER[0], 'post', int(post)]) if dev is None: postdev.append(FINAL_MARKER[0]) if dev is not None: postdev.extend(['dev', int(dev)]) parts.append(tuple(postdev)) else: parts.append(FINAL_MARKER) self.parts = tuple(parts) if error_on_huge_major_num and self.parts[0][0] > 1980: raise HugeMajorVersionNumError("huge major version number, %r, " "which might cause future problems: %r" % (self.parts[0][0], s)) def _parse_numdots(self, s, full_ver_str, drop_trailing_zeros=True, pad_zeros_length=0): """Parse 'N.N.N' sequences, return a list of ints. @param s {str} 'N.N.N...' sequence to be parsed @param full_ver_str {str} The full version string from which this comes. Used for error strings. @param drop_trailing_zeros {bool} Whether to drop trailing zeros from the returned list. Default True. @param pad_zeros_length {int} The length to which to pad the returned list with zeros, if necessary. Default 0. """ nums = [] for n in s.split("."): if len(n) > 1 and n[0] == '0': raise IrrationalVersionError("cannot have leading zero in " "version number segment: '%s' in %r" % (n, full_ver_str)) nums.append(int(n)) if drop_trailing_zeros: while nums and nums[-1] == 0: nums.pop() while len(nums) < pad_zeros_length: nums.append(0) return nums def __str__(self): return self.parts_to_str(self.parts) @classmethod def parts_to_str(cls, parts): """Transforms a version expressed in tuple into its string representation.""" # XXX This doesn't check for invalid tuples main, prerel, postdev = parts s = '.'.join(str(v) for v in main) if prerel is not FINAL_MARKER: s += prerel[0] s += '.'.join(str(v) for v in prerel[1:]) if postdev and postdev is not FINAL_MARKER: if postdev[0] == 'f': postdev = postdev[1:] i = 0 while i < len(postdev): if i % 2 == 0: s += '.' s += str(postdev[i]) i += 1 return s def __repr__(self): return "%s('%s')" % (self.__class__.__name__, self) def _cannot_compare(self, other): raise TypeError("cannot compare %s and %s" % (type(self).__name__, type(other).__name__)) def __eq__(self, other): if not isinstance(other, NormalizedVersion): self._cannot_compare(other) return self.parts == other.parts def __lt__(self, other): if not isinstance(other, NormalizedVersion): self._cannot_compare(other) return self.parts < other.parts def __ne__(self, other): return not self.__eq__(other) def __gt__(self, other): return not (self.__lt__(other) or self.__eq__(other)) def __le__(self, other): return self.__eq__(other) or self.__lt__(other) def __ge__(self, other): return self.__eq__(other) or self.__gt__(other) def suggest_normalized_version(s): """Suggest a normalized version close to the given version string. If you have a version string that isn't rational (i.e. NormalizedVersion doesn't like it) then you might be able to get an equivalent (or close) rational version from this function. This does a number of simple normalizations to the given string, based on observation of versions currently in use on PyPI. Given a dump of those version during PyCon 2009, 4287 of them: - 2312 (53.93%) match NormalizedVersion without change - with the automatic suggestion - 3474 (81.04%) match when using this suggestion method @param s {str} An irrational version string. @returns A rational version string, or None, if couldn't determine one. """ try: NormalizedVersion(s) return s # already rational except IrrationalVersionError: pass rs = s.lower() # part of this could use maketrans for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'), ('beta', 'b'), ('rc', 'c'), ('-final', ''), ('-pre', 'c'), ('-release', ''), ('.release', ''), ('-stable', ''), ('+', '.'), ('_', '.'), (' ', ''), ('.final', ''), ('final', '')): rs = rs.replace(orig, repl) # if something ends with dev or pre, we add a 0 rs = re.sub(r"pre$", r"pre0", rs) rs = re.sub(r"dev$", r"dev0", rs) # if we have something like "b-2" or "a.2" at the end of the # version, that is pobably beta, alpha, etc # let's remove the dash or dot rs = re.sub(r"([abc|rc])[\-\.](\d+)$", r"\1\2", rs) # 1.0-dev-r371 -> 1.0.dev371 # 0.1-dev-r79 -> 0.1.dev79 rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs) # Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1 rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs) # Clean: v0.3, v1.0 if rs.startswith('v'): rs = rs[1:] # Clean leading '0's on numbers. #TODO: unintended side-effect on, e.g., "2003.05.09" # PyPI stats: 77 (~2%) better rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) # Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers # zero. # PyPI stats: 245 (7.56%) better rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs) # the 'dev-rNNN' tag is a dev tag rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs) # clean the - when used as a pre delimiter rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs) # a terminal "dev" or "devel" can be changed into ".dev0" rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs) # a terminal "dev" can be changed into ".dev0" rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs) # a terminal "final" or "stable" can be removed rs = re.sub(r"(final|stable)$", "", rs) # The 'r' and the '-' tags are post release tags # 0.4a1.r10 -> 0.4a1.post10 # 0.9.33-17222 -> 0.9.3.post17222 # 0.9.33-r17222 -> 0.9.3.post17222 rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs) # Clean 'r' instead of 'dev' usage: # 0.9.33+r17222 -> 0.9.3.dev17222 # 1.0dev123 -> 1.0.dev123 # 1.0.git123 -> 1.0.dev123 # 1.0.bzr123 -> 1.0.dev123 # 0.1a0dev.123 -> 0.1a0.dev123 # PyPI stats: ~150 (~4%) better rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs) # Clean '.pre' (normalized from '-pre' above) instead of 'c' usage: # 0.2.pre1 -> 0.2c1 # 0.2-c1 -> 0.2c1 # 1.0preview123 -> 1.0c123 # PyPI stats: ~21 (0.62%) better rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs) # Tcl/Tk uses "px" for their post release markers rs = re.sub(r"p(\d+)$", r".post\1", rs) try: NormalizedVersion(rs) return rs # already rational except IrrationalVersionError: pass return None pyutil-3.3.2/pyutil/version_class.py000066400000000000000000000125711437014040400176220ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ extended version number class """ # verlib a.k.a. distutils.version by Tarek Ziadé. from pyutil.verlib import NormalizedVersion # Python Standard Library import functools, re # End users see version strings like this: # "1.0.0" # ^ ^ ^ # | | | # | | '- micro version number # | '- minor version number # '- major version number # The first number is "major version number". The second number is the "minor # version number" -- it gets bumped whenever we make a new release that adds or # changes functionality. The third version is the "micro version number" -- it # gets bumped whenever we make a new release that doesn't add or change # functionality, but just fixes bugs (including performance issues). # Early-adopter end users see version strings like this: # "1.0.0a1" # ^ ^ ^^^ # | | ||| # | | ||'- release number # | | |'- a=alpha, b=beta, c=release candidate, or none # | | '- micro version number # | '- minor version number # '- major version number # The optional "a" or "b" stands for "alpha release" or "beta release" # respectively. The number after "a" or "b" gets bumped every time we # make a new alpha or beta release. This has the same form and the same # meaning as version numbers of releases of Python. # Developers see "full version strings", like this: # "1.0.0a1-55" # ^ ^ ^^^ ^ # | | ||| | # | | ||| '- nano version number # | | ||'- release number # | | |'- a=alpha, b=beta, c=release candidate or none # | | '- micro version number # | '- minor version number # '- major version number # or else like this: # "1.0.0a1-r22155" # ^ ^ ^^^ ^ # | | ||| | # | | ||| '- revision number # | | ||'- release number # | | |'- a=alpha, b=beta, c=release candidate or none # | | '- micro version number # | '- minor version number # '- major version number # The presence of the nano version number means that this is a development # version. There are no guarantees about compatibility, etc. This version is # considered to be more recent than the version without this field # (e.g. "1.0.0a1"). # The nano version number or revision number is meaningful only to developers. # It gets generated automatically from darcs revision control history by # "darcsver.py". The nano version number is the count of patches that have been # applied since the last version number tag was applied. The revision number is # the count of all patches that have been applied in the history. VERSION_BASE_RE_STR="(\d+)(\.(\d+)(\.(\d+))?)?((a|b|c)(\d+))?(\.dev(\d+))?" VERSION_SUFFIX_RE_STR="(-(\d+|r\d+)|.post\d+)?" VERSION_RE_STR=VERSION_BASE_RE_STR + VERSION_SUFFIX_RE_STR VERSION_RE=re.compile("^" + VERSION_RE_STR + "$") @functools.total_ordering class Version(object): def __init__(self, vstring=None): self.major = None self.minor = None self.micro = None self.prereleasetag = None self.prerelease = None self.nano = None self.revision = None if vstring: try: self.parse(vstring) except ValueError as le: le.args = tuple(le.args + ('vstring:', vstring,)) raise def parse(self, vstring): mo = VERSION_RE.search(vstring) if not mo: raise ValueError("Not a valid version string for pyutil.version_class.Version(): %r" % (vstring,)) self.major = int(mo.group(1)) self.minor = mo.group(3) and int(mo.group(3)) or 0 self.micro = mo.group(5) and int(mo.group(5)) or 0 reltag = mo.group(6) if reltag: reltagnum = int(mo.group(8)) self.prereleasetag = mo.group(7) self.prerelease = reltagnum if mo.group(11): if mo.group(11)[0] == '-': if mo.group(12)[0] == 'r': self.revision = int(mo.group(12)[1:]) else: self.nano = int(mo.group(12)) else: assert mo.group(11).startswith('.post'), mo.group(11) self.revision = int(mo.group(11)[5:]) # XXX in the future, to be compatible with the Python "rational version numbering" scheme, we should move to using .post$REV instead of -r$REV: # self.fullstr = "%d.%d.%d%s%s" % (self.major, self.minor, self.micro, self.prereleasetag and "%s%d" % (self.prereleasetag, self.prerelease,) or "", self.nano and "-%d" % (self.nano,) or self.revision and ".post%d" % (self.revision,) or "",) self.fullstr = "%d.%d.%d%s%s" % (self.major, self.minor, self.micro, self.prereleasetag and "%s%d" % (self.prereleasetag, self.prerelease,) or "", self.nano and "-%d" % (self.nano,) or self.revision and "-r%d" % (self.revision,) or "",) def user_str(self): return self.full_str() def full_str(self): if hasattr(self, 'fullstr'): return self.fullstr else: return 'None' def __str__(self): return self.full_str() def __repr__(self): return self.__str__() def __cmp__ (self, other): return cmp(NormalizedVersion(str(self)), NormalizedVersion(str(other))) def __eq__ (self, other): return NormalizedVersion(str(self)) == NormalizedVersion(str(other)) def __lt__ (self, other): return NormalizedVersion(str(self)) < NormalizedVersion(str(other)) pyutil-3.3.2/pyutil/weakutil.py000066400000000000000000000026271437014040400165760ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. import warnings # from the Python Standard Library from weakref import ref import types # from the pyutil library from .assertutil import precondition def is_bound_method(fn): if hasattr(types, 'UnboundMethodType'): # PY2 return hasattr(fn, 'im_self') else: # PY3 return isinstance(fn, types.MethodType) class WeakMethod: """ Wraps a function or, more importantly, a bound method, in a way that allows a bound method's object to be GC'd """ def __init__(self, fn, callback=None): warnings.warn("deprecated", DeprecationWarning) precondition(is_bound_method(fn), "fn is required to be a bound method.") self._cleanupcallback = callback self._obj = ref(fn.__self__, self.call_cleanup_cb) self._meth = fn.__func__ def __call__(self, *args, **kws): s = self._obj() if s: return self._meth(s, *args,**kws) def __repr__(self): return "<%s %s %s>" % (self.__class__.__name__, self._obj, self._meth,) def call_cleanup_cb(self, thedeadweakref): if self._cleanupcallback is not None: self._cleanupcallback(self, thedeadweakref) def factory_function_name_here(o): if is_bound_method(o): return WeakMethod(o) else: return o pyutil-3.3.2/pyutil/xor/000077500000000000000000000000001437014040400152005ustar00rootroot00000000000000pyutil-3.3.2/pyutil/xor/__init__.py000066400000000000000000000000001437014040400172770ustar00rootroot00000000000000pyutil-3.3.2/pyutil/xor/xor.py000066400000000000000000000034111437014040400163610ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ What word has three letters and a 'x' in it? Not that one silly. """ import warnings import array, operator from pyutil.assertutil import precondition def py_xor(str1, str2): warnings.warn("deprecated", DeprecationWarning) precondition(len(str1) == len(str2), "str1 and str2 are required to be of the same length.", str1=str1, str2=str2) if len(str1)%4 == 0: a1 = array.array('i', str1) a2 = array.array('i', str2) for i in range(len(a1)): a2[i] = a2[i]^a1[i] elif len(str1)%2 == 0: a1 = array.array('h', str1) a2 = array.array('h', str2) for i in range(len(a1)): a2[i] = a2[i]^a1[i] else: a1 = array.array('b', str1) a2 = array.array('b', str2) for i in range(len(a1)): a2[i] = a2[i]^a1[i] if hasattr(a2, 'tobytes'): # PY3 return a2.tobytes() else: # PY2 return a2.tostring() def py_xor_simple(str1, str2): """ Benchmarks show that this is the same speed as py_xor() for small strings and much slower for large strings, so don't use it. --Zooko 2002-04-29 """ warnings.warn("deprecated", DeprecationWarning) precondition(len(str1) == len(str2), "str1 and str2 are required to be of the same length.", str1=str1, str2=str2) if bytes != str: # PY3 return bytes(map(operator.__xor__, str1, str2)) else: # PY2 return ''.join(map(chr, map(operator.__xor__, map(ord, str1), map(ord, str2)))) # Now make "xor.xor()" be the best xor we've got: xor = py_xor # for unit tests, see pyutil/test/test_xor.py. For benchmarks, see pyutil/test/bench_xor.py. pyutil-3.3.2/pyutil/zlibutil.py000066400000000000000000000372541437014040400166130ustar00rootroot00000000000000# -*- coding: utf-8; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # This file is part of pyutil; see README.rst for licensing terms. """ Making your zlib experience that much nicer! Most importantly, this offers protection from "zlib bomb" attacks, where the original data was maximally compressable, and a naive use of zlib would consume all of your RAM while trying to decompress it. """ import string, zlib import numbers try: StandardError # PY2 except NameError: StandardError = Exception # PY3 from .humanreadable import hr from .assertutil import precondition class DecompressError(zlib.error, StandardError): pass class UnsafeDecompressError(DecompressError): pass # This means it would take more memory to decompress than we can spare. class TooBigError(DecompressError): pass # This means the resulting uncompressed text would exceed the maximum allowed length. class ZlibError(DecompressError): pass # internal error, probably due to the input not being zlib compressed text # The smallest limit that you can impose on zlib decompression and still have # a chance of succeeding at decompression. # constant memory overhead of zlib (76 KB), plus minbite (128 bytes) times # maxexpansion (1032) times buffer-copying duplication (2), plus 2063 so as # to reach the ceiling of div (2*1032) MINMAXMEM=76*2**10 + 128 * 1032 * 2 + 2063 - 1 # You should really specify a maxmem which is much higher than MINMAXMEM. If # maxmem=MINMAXMEM, we will be reduced to decompressing the input in # 128-byte bites, and furthermore unless the decompressed text is quite small, # we will be forced to give up and spuriously raise UnsafeDecompressError! # You really ought to pass a maxmem argument equal to the maximum possible # memory that your app should ever allocate (for a short-term use). # I typically set it to 65 MB. def decompress(zbuf, maxlen=(65 * (2**20)), maxmem=(65 * (2**20))): """ Decompress zbuf so that it decompresses to <= maxlen bytes, while using <= maxmem memory, or else raise an exception. If zbuf contains uncompressed data an exception will be raised. This function guards against memory allocation attacks. @param maxlen the resulting text must not be greater than this @param maxmem the execution of this function must not use more than this amount of memory in bytes; The higher this number is (optimally 1032 * maxlen, or even greater), the faster this function can complete. (Actually I don't fully understand the workings of zlib, so this function might use a *little* more than this memory, but not a lot more.) (Also, this function will raise an exception if the amount of memory required even *approaches* maxmem. Another reason to make it large.) (Hence the default value which would seem to be exceedingly large until you realize that it means you can decompress 64 KB chunks of compressiontext at a bite.) """ assert isinstance(maxlen, numbers.Integral) and maxlen > 0, "maxlen is required to be a real maxlen, geez!" assert isinstance(maxmem, numbers.Integral) and maxmem > 0, "maxmem is required to be a real maxmem, geez!" assert maxlen <= maxmem, "maxlen is required to be <= maxmem. All data that is included in the return value is counted against maxmem as well as against maxlen, so it is impossible to return a result bigger than maxmem, even if maxlen is bigger than maxmem. See decompress_to_spool() if you want to spool a large text out while limiting the amount of memory used during the process." lenzbuf = len(zbuf) offset = 0 decomplen = 0 availmem = maxmem - (76 * 2**10) # zlib can take around 76 KB RAM to do decompression availmem = availmem / 2 # generating the result string from the intermediate strings will require using the same amount of memory again, briefly. If you care about this kind of thing, then let's rewrite this module in C. decompstrlist = [] decomp = zlib.decompressobj() while offset < lenzbuf: # How much compressedtext can we safely attempt to decompress now without going over `maxmem'? zlib docs say that theoretical maximum for the zlib format would be 1032:1. lencompbite = availmem / 1032 # XXX TODO: The biggest compression ratio zlib can have for whole files is 1032:1. Unfortunately I don't know if small chunks of compressiontext *within* a file can expand to more than that. I'll assume not... --Zooko 2001-05-12 if lencompbite < 128: # If we can't safely attempt even a few bytes of compression text, let us give up. Either `maxmem' was too small or this compressiontext is actually a decompression bomb. raise UnsafeDecompressError("used up roughly maxmem memory. maxmem: %s, len(zbuf): %s, offset: %s, decomplen: %s, lencompbite: %s" % tuple(map(hr, [maxmem, len(zbuf), offset, decomplen, lencompbite,]))) # I wish the following were a local function like this: # def proc_decomp_bite(tmpstr, lencompbite=0, decomplen=decomplen, maxlen=maxlen, availmem=availmem, decompstrlist=decompstrlist, offset=offset, zbuf=zbuf): # ...but we can't conveniently and efficiently update the integer variables like offset in the outer scope. Oh well. --Zooko 2003-06-26 try: if (offset == 0) and (lencompbite >= lenzbuf): tmpstr = decomp.decompress(zbuf) else: tmpstr = decomp.decompress(zbuf[offset:offset+lencompbite]) except zlib.error as le: raise ZlibError((offset, lencompbite, decomplen, hr(le), )) lentmpstr = len(tmpstr) decomplen = decomplen + lentmpstr if decomplen > maxlen: raise TooBigError("length of resulting data > maxlen. maxlen: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxlen, len(zbuf), offset, decomplen,]))) availmem = availmem - lentmpstr offset = offset + lencompbite decompstrlist.append(tmpstr) tmpstr = '' try: tmpstr = decomp.flush() except zlib.error as le: raise ZlibError((offset, lencompbite, decomplen, le, )) lentmpstr = len(tmpstr) decomplen = decomplen + lentmpstr if decomplen > maxlen: raise TooBigError("length of resulting data > maxlen. maxlen: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxlen, len(zbuf), offset, decomplen,]))) availmem = availmem - lentmpstr offset = offset + lencompbite if lentmpstr > 0: decompstrlist.append(tmpstr) tmpstr = '' if len(decompstrlist) > 0: return b''.join(decompstrlist) else: return decompstrlist[0] def decompress_to_fileobj(zbuf, fileobj, maxlen=(65 * (2**20)), maxmem=(65 * (2**20))): """ Decompress zbuf so that it decompresses to <= maxlen bytes, while using <= maxmem memory, or else raise an exception. If zbuf contains uncompressed data an exception will be raised. This function guards against memory allocation attacks. Note that this assumes that data written to fileobj still occupies memory, so such data counts against maxmem as well as against maxlen. @param maxlen the resulting text must not be greater than this @param maxmem the execution of this function must not use more than this amount of memory in bytes; The higher this number is (optimally 1032 * maxlen, or even greater), the faster this function can complete. (Actually I don't fully understand the workings of zlib, so this function might use a *little* more than this memory, but not a lot more.) (Also, this function will raise an exception if the amount of memory required even *approaches* maxmem. Another reason to make it large.) (Hence the default value which would seem to be exceedingly large until you realize that it means you can decompress 64 KB chunks of compressiontext at a bite.) @param fileobj a file object to which the decompressed text will be written """ precondition(hasattr(fileobj, 'write') and callable(fileobj.write), "fileobj is required to have a write() method.", fileobj=fileobj) precondition(isinstance(maxlen, numbers.Integral) and maxlen > 0, "maxlen is required to be a real maxlen, geez!", maxlen=maxlen) precondition(isinstance(maxmem, numbers.Integral) and maxmem > 0, "maxmem is required to be a real maxmem, geez!", maxmem=maxmem) precondition(maxlen <= maxmem, "maxlen is required to be <= maxmem. All data that is written out to fileobj is counted against maxmem as well as against maxlen, so it is impossible to return a result bigger than maxmem, even if maxlen is bigger than maxmem. See decompress_to_spool() if you want to spool a large text out while limiting the amount of memory used during the process.", maxlen=maxlen, maxmem=maxmem) lenzbuf = len(zbuf) offset = 0 decomplen = 0 availmem = maxmem - (76 * 2**10) # zlib can take around 76 KB RAM to do decompression decomp = zlib.decompressobj() while offset < lenzbuf: # How much compressedtext can we safely attempt to decompress now without going over maxmem? zlib docs say that theoretical maximum for the zlib format would be 1032:1. lencompbite = availmem / 1032 # XXX TODO: The biggest compression ratio zlib can have for whole files is 1032:1. Unfortunately I don't know if small chunks of compressiontext *within* a file can expand to more than that. I'll assume not... --Zooko 2001-05-12 if lencompbite < 128: # If we can't safely attempt even a few bytes of compression text, let us give up. Either maxmem was too small or this compressiontext is actually a decompression bomb. raise UnsafeDecompressError("used up roughly maxmem memory. maxmem: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxmem, len(zbuf), offset, decomplen,]))) # I wish the following were a local function like this: # def proc_decomp_bite(tmpstr, lencompbite=0, decomplen=decomplen, maxlen=maxlen, availmem=availmem, decompstrlist=decompstrlist, offset=offset, zbuf=zbuf): # ...but we can't conveniently and efficiently update the integer variables like offset in the outer scope. Oh well. --Zooko 2003-06-26 try: if (offset == 0) and (lencompbite >= lenzbuf): tmpstr = decomp.decompress(zbuf) else: tmpstr = decomp.decompress(zbuf[offset:offset+lencompbite]) except zlib.error as le: raise ZlibError((offset, lencompbite, decomplen, le, )) lentmpstr = len(tmpstr) decomplen = decomplen + lentmpstr if decomplen > maxlen: raise TooBigError("length of resulting data > maxlen. maxlen: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxlen, len(zbuf), offset, decomplen,]))) availmem = availmem - lentmpstr offset = offset + lencompbite fileobj.write(tmpstr) tmpstr = '' try: tmpstr = decomp.flush() except zlib.error as le: raise ZlibError((offset, lencompbite, decomplen, le, )) lentmpstr = len(tmpstr) decomplen = decomplen + lentmpstr if decomplen > maxlen: raise TooBigError("length of resulting data > maxlen. maxlen: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxlen, len(zbuf), offset, decomplen,]))) availmem = availmem - lentmpstr offset = offset + lencompbite fileobj.write(tmpstr) tmpstr = '' def decompress_to_spool(zbuf, fileobj, maxlen=(65 * (2**20)), maxmem=(65 * (2**20))): """ Decompress zbuf so that it decompresses to <= maxlen bytes, while using <= maxmem memory, or else raise an exception. If zbuf contains uncompressed data an exception will be raised. This function guards against memory allocation attacks. Note that this assumes that data written to fileobj does *not* continue to occupy memory, so such data doesn't count against maxmem, although of course it still counts against maxlen. @param maxlen the resulting text must not be greater than this @param maxmem the execution of this function must not use more than this amount of memory in bytes; The higher this number is (optimally 1032 * maxlen, or even greater), the faster this function can complete. (Actually I don't fully understand the workings of zlib, so this function might use a *little* more than this memory, but not a lot more.) (Also, this function will raise an exception if the amount of memory required even *approaches* maxmem. Another reason to make it large.) (Hence the default value which would seem to be exceedingly large until you realize that it means you can decompress 64 KB chunks of compressiontext at a bite.) @param fileobj the decompressed text will be written to it """ precondition(hasattr(fileobj, 'write') and callable(fileobj.write), "fileobj is required to have a write() method.", fileobj=fileobj) precondition(isinstance(maxlen, numbers.Integral) and maxlen > 0, "maxlen is required to be a real maxlen, geez!", maxlen=maxlen) precondition(isinstance(maxmem, numbers.Integral) and maxmem > 0, "maxmem is required to be a real maxmem, geez!", maxmem=maxmem) tmpstr = '' lenzbuf = len(zbuf) offset = 0 decomplen = 0 availmem = maxmem - (76 * 2**10) # zlib can take around 76 KB RAM to do decompression decomp = zlib.decompressobj() while offset < lenzbuf: # How much compressedtext can we safely attempt to decompress now without going over `maxmem'? zlib docs say that theoretical maximum for the zlib format would be 1032:1. lencompbite = availmem / 1032 # XXX TODO: The biggest compression ratio zlib can have for whole files is 1032:1. Unfortunately I don't know if small chunks of compressiontext *within* a file can expand to more than that. I'll assume not... --Zooko 2001-05-12 if lencompbite < 128: # If we can't safely attempt even a few bytes of compression text, let us give up. Either `maxmem' was too small or this compressiontext is actually a decompression bomb. raise UnsafeDecompressError("used up roughly `maxmem' memory. maxmem: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxmem, len(zbuf), offset, decomplen,]))) # I wish the following were a local function like this: # def proc_decomp_bite(tmpstr, lencompbite=0, decomplen=decomplen, maxlen=maxlen, availmem=availmem, decompstrlist=decompstrlist, offset=offset, zbuf=zbuf): # ...but we can't conveniently and efficiently update the integer variables like offset in the outer scope. Oh well. --Zooko 2003-06-26 try: if (offset == 0) and (lencompbite >= lenzbuf): tmpstr = decomp.decompress(zbuf) else: tmpstr = decomp.decompress(zbuf[offset:offset+lencompbite]) except zlib.error as le: raise ZlibError((offset, lencompbite, decomplen, le, )) lentmpstr = len(tmpstr) decomplen = decomplen + lentmpstr if decomplen > maxlen: raise TooBigError("length of resulting data > `maxlen'. maxlen: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxlen, len(zbuf), offset, decomplen,]))) offset = offset + lencompbite fileobj.write(tmpstr) tmpstr = '' try: tmpstr = decomp.flush() except zlib.error as le: raise ZlibError((offset, lencompbite, decomplen, le, )) lentmpstr = len(tmpstr) decomplen = decomplen + lentmpstr if decomplen > maxlen: raise TooBigError("length of resulting data > `maxlen'. maxlen: %s, len(zbuf): %s, offset: %s, decomplen: %s" % tuple(map(hr, [maxlen, len(zbuf), offset, decomplen,]))) offset = offset + lencompbite fileobj.write(tmpstr) tmpstr = '' pyutil-3.3.2/setup.cfg000066400000000000000000000010471437014040400146650ustar00rootroot00000000000000[easy_install] # pyutil actually does work at least as well as any package works # when zipped, but zipping eggs causes various problems # (http://bugs.python.org/setuptools/issue33 ), and generally makes it # harder for people to get at the source code, and doesn't actually # provide any benefits that I am aware of. zip_ok = False [egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 [versioneer] VCS = git style = pep440 versionfile_source = pyutil/_version.py versionfile_build = pyutil/_version.py tag_prefix = parentdir_prefix = pyutil- pyutil-3.3.2/setup.py000066400000000000000000000060561437014040400145630ustar00rootroot00000000000000#!/usr/bin/env python # -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- # -*- indent-tabs-mode: nil -*- # pyutil -- utility functions and classes # # This file is part of pyutil; see README.rst for licensing terms. import os, io, re, sys import versioneer from setuptools import find_packages, setup trove_classifiers=[ u"Development Status :: 5 - Production/Stable", u"License :: OSI Approved :: GNU General Public License (GPL)", u"License :: DFSG approved", u"Intended Audience :: Developers", u"Operating System :: Microsoft :: Windows", u"Operating System :: Unix", u"Operating System :: MacOS :: MacOS X", u"Operating System :: OS Independent", u"Natural Language :: English", u"Programming Language :: Python", u"Programming Language :: Python :: 2", u"Programming Language :: Python :: 2.7", u"Programming Language :: Python :: 3", u"Programming Language :: Python :: 3.5", u"Programming Language :: Python :: 3.6", u"Programming Language :: Python :: 3.7", u"Topic :: Utilities", u"Topic :: Software Development :: Libraries", ] PKG=u'pyutil' doc_fnames=[ u'COPYING.SPL.txt', u'COPYING.GPL', u'COPYING.TGPPL.rst', u'README.rst', u'CREDITS' ] # In case we are building for a .deb with stdeb's sdist_dsc command, we put the # docs in "share/doc/python-$PKG". doc_loc = u"share/doc/" + PKG data_files = [ (doc_loc, doc_fnames), (os.path.join(u'pyutil', u'data'), [os.path.join(u'pyutil', u'data', u'wordlist.txt')]) ] readmetext = io.open(u'README.rst', encoding='utf-8').read() setup(name=PKG, version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), description=u'a collection of utilities for Python programmers', long_description=readmetext, long_description_content_type='text/x-rst', author=u"tpltnt", author_email=u'tpltnt+pyutil@nkawtg.net', url=u'https://github.com/tpltnt/' + PKG, license=u'GNU GPL', # see README.rst for details -- there are also alternative licences packages=find_packages(), include_package_data=True, data_files=data_files, install_requires=[], extras_require={ u'jsonutil': [u'simplejson >= 2.1.0',], u'randcookie': [u'zbase32 >= 1.0',], }, tests_require=[ u'twisted >= 15.5.0', # for trial (eg user: test_observer) u'mock >= 1.3.0', ], classifiers=trove_classifiers, entry_points = { u'console_scripts': [ u'randcookie = pyutil.scripts.randcookie:main', u'tailx = pyutil.scripts.tailx:main', u'lines = pyutil.scripts.lines:main', u'randfile = pyutil.scripts.randfile:main', u'unsort = pyutil.scripts.unsort:main', u'verinfo = pyutil.scripts.verinfo:main', u'try_decoding = pyutil.scripts.try_decoding:main', u'passphrase = pyutil.scripts.passphrase:main', ] }, test_suite=PKG+u".test", zip_safe=False, # I prefer unzipped for easier access. ) pyutil-3.3.2/versioneer.py000066400000000000000000002432271437014040400156070ustar00rootroot00000000000000 # Version: 0.28 """The Versioneer - like a rocketeer, but for versions. The Versioneer ============== * like a rocketeer, but for versions! * https://github.com/python-versioneer/python-versioneer * Brian Warner * License: Public Domain (Unlicense) * Compatible with: Python 3.7, 3.8, 3.9, 3.10 and pypy3 * [![Latest Version][pypi-image]][pypi-url] * [![Build Status][travis-image]][travis-url] This is a tool for managing a recorded version number in setuptools-based python projects. The goal is to remove the tedious and error-prone "update the embedded version string" step from your release process. Making a new release should be as easy as recording a new tag in your version-control system, and maybe making new tarballs. ## Quick Install Versioneer provides two installation modes. The "classic" vendored mode installs a copy of versioneer into your repository. The experimental build-time dependency mode is intended to allow you to skip this step and simplify the process of upgrading. ### Vendored mode * `pip install versioneer` to somewhere in your $PATH * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is available, so you can also use `conda install -c conda-forge versioneer` * add a `[tool.versioneer]` section to your `pyproject.toml` or a `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) * Note that you will need to add `tomli; python_version < "3.11"` to your build-time dependencies if you use `pyproject.toml` * run `versioneer install --vendor` in your source tree, commit the results * verify version information with `python setup.py version` ### Build-time dependency mode * `pip install versioneer` to somewhere in your $PATH * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is available, so you can also use `conda install -c conda-forge versioneer` * add a `[tool.versioneer]` section to your `pyproject.toml` or a `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) * add `versioneer` (with `[toml]` extra, if configuring in `pyproject.toml`) to the `requires` key of the `build-system` table in `pyproject.toml`: ```toml [build-system] requires = ["setuptools", "versioneer[toml]"] build-backend = "setuptools.build_meta" ``` * run `versioneer install --no-vendor` in your source tree, commit the results * verify version information with `python setup.py version` ## Version Identifiers Source trees come from a variety of places: * a version-control system checkout (mostly used by developers) * a nightly tarball, produced by build automation * a snapshot tarball, produced by a web-based VCS browser, like github's "tarball from tag" feature * a release tarball, produced by "setup.py sdist", distributed through PyPI Within each source tree, the version identifier (either a string or a number, this tool is format-agnostic) can come from a variety of places: * ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows about recent "tags" and an absolute revision-id * the name of the directory into which the tarball was unpacked * an expanded VCS keyword ($Id$, etc) * a `_version.py` created by some earlier build step For released software, the version identifier is closely related to a VCS tag. Some projects use tag names that include more than just the version string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool needs to strip the tag prefix to extract the version identifier. For unreleased software (between tags), the version identifier should provide enough information to help developers recreate the same tree, while also giving them an idea of roughly how old the tree is (after version 1.2, before version 1.3). Many VCS systems can report a description that captures this, for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has uncommitted changes). The version identifier is used for multiple purposes: * to allow the module to self-identify its version: `myproject.__version__` * to choose a name and prefix for a 'setup.py sdist' tarball ## Theory of Operation Versioneer works by adding a special `_version.py` file into your source tree, where your `__init__.py` can import it. This `_version.py` knows how to dynamically ask the VCS tool for version information at import time. `_version.py` also contains `$Revision$` markers, and the installation process marks `_version.py` to have this marker rewritten with a tag name during the `git archive` command. As a result, generated tarballs will contain enough information to get the proper version. To allow `setup.py` to compute a version too, a `versioneer.py` is added to the top level of your source tree, next to `setup.py` and the `setup.cfg` that configures it. This overrides several distutils/setuptools commands to compute the version when invoked, and changes `setup.py build` and `setup.py sdist` to replace `_version.py` with a small static file that contains just the generated version data. ## Installation See [INSTALL.md](./INSTALL.md) for detailed installation instructions. ## Version-String Flavors Code which uses Versioneer can learn about its version string at runtime by importing `_version` from your main `__init__.py` file and running the `get_versions()` function. From the "outside" (e.g. in `setup.py`), you can import the top-level `versioneer.py` and run `get_versions()`. Both functions return a dictionary with different flavors of version information: * `['version']`: A condensed version string, rendered using the selected style. This is the most commonly used value for the project's version string. The default "pep440" style yields strings like `0.11`, `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section below for alternative styles. * `['full-revisionid']`: detailed revision identifier. For Git, this is the full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". * `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the commit date in ISO 8601 format. This will be None if the date is not available. * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that this is only accurate if run in a VCS checkout, otherwise it is likely to be False or None * `['error']`: if the version string could not be computed, this will be set to a string describing the problem, otherwise it will be None. It may be useful to throw an exception in setup.py if this is set, to avoid e.g. creating tarballs with a version string of "unknown". Some variants are more useful than others. Including `full-revisionid` in a bug report should allow developers to reconstruct the exact code being tested (or indicate the presence of local changes that should be shared with the developers). `version` is suitable for display in an "about" box or a CLI `--version` output: it can be easily compared against release notes and lists of bugs fixed in various releases. The installer adds the following text to your `__init__.py` to place a basic version in `YOURPROJECT.__version__`: from ._version import get_versions __version__ = get_versions()['version'] del get_versions ## Styles The setup.cfg `style=` configuration controls how the VCS information is rendered into a version string. The default style, "pep440", produces a PEP440-compliant string, equal to the un-prefixed tag name for actual releases, and containing an additional "local version" section with more detail for in-between builds. For Git, this is TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags --dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and that this commit is two revisions ("+2") beyond the "0.11" tag. For released software (exactly equal to a known tag), the identifier will only contain the stripped tag, e.g. "0.11". Other styles are available. See [details.md](details.md) in the Versioneer source tree for descriptions. ## Debugging Versioneer tries to avoid fatal errors: if something goes wrong, it will tend to return a version of "0+unknown". To investigate the problem, run `setup.py version`, which will run the version-lookup code in a verbose mode, and will display the full contents of `get_versions()` (including the `error` string, which may help identify what went wrong). ## Known Limitations Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github [issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects Versioneer has limited support for source trees in which `setup.py` is not in the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are two common reasons why `setup.py` might not be in the root: * Source trees which contain multiple subprojects, such as [Buildbot](https://github.com/buildbot/buildbot), which contains both "master" and "slave" subprojects, each with their own `setup.py`, `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs and implementation details which frequently cause `pip install .` from a subproject directory to fail to find a correct version string (so it usually defaults to `0+unknown`). `pip install --editable .` should work correctly. `setup.py install` might work too. Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. [Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in [PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve pip to let Versioneer work correctly. Versioneer-0.16 and earlier only looked for a `.git` directory next to the `setup.cfg`, so subprojects were completely unsupported with those releases. ### Editable installs with setuptools <= 18.5 `setup.py develop` and `pip install --editable .` allow you to install a project into a virtualenv once, then continue editing the source code (and test) without re-installing after every change. "Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a convenient way to specify executable scripts that should be installed along with the python package. These both work as expected when using modern setuptools. When using setuptools-18.5 or earlier, however, certain operations will cause `pkg_resources.DistributionNotFound` errors when running the entrypoint script, which must be resolved by re-installing the package. This happens when the install happens with one version, then the egg_info data is regenerated while a different version is checked out. Many setup.py commands cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. [Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. ## Updating Versioneer To upgrade your project to a new release of Versioneer, do the following: * install the new Versioneer (`pip install -U versioneer` or equivalent) * edit `setup.cfg` and `pyproject.toml`, if necessary, to include any new configuration settings indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. * re-run `versioneer install --[no-]vendor` in your source tree, to replace `SRC/_version.py` * commit any changed files ## Future Directions This tool is designed to make it easily extended to other version-control systems: all VCS-specific components are in separate directories like src/git/ . The top-level `versioneer.py` script is assembled from these components by running make-versioneer.py . In the future, make-versioneer.py will take a VCS name as an argument, and will construct a version of `versioneer.py` that is specific to the given VCS. It might also take the configuration arguments that are currently provided manually during installation by editing setup.py . Alternatively, it might go the other direction and include code from all supported VCS systems, reducing the number of intermediate scripts. ## Similar projects * [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time dependency * [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of versioneer * [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools plugin ## License To make Versioneer easier to embed, all its code is dedicated to the public domain. The `_version.py` that it creates is also in the public domain. Specifically, both are released under the "Unlicense", as described in https://unlicense.org/. [pypi-image]: https://img.shields.io/pypi/v/versioneer.svg [pypi-url]: https://pypi.python.org/pypi/versioneer/ [travis-image]: https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg [travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer """ # pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring # pylint:disable=missing-class-docstring,too-many-branches,too-many-statements # pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error # pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with # pylint:disable=attribute-defined-outside-init,too-many-arguments import configparser import errno import json import os import re import subprocess import sys from pathlib import Path from typing import Callable, Dict import functools have_tomllib = True if sys.version_info >= (3, 11): import tomllib else: try: import tomli as tomllib except ImportError: have_tomllib = False class VersioneerConfig: """Container for Versioneer configuration parameters.""" def get_root(): """Get the project root directory. We require that all commands are run from the project root, i.e. the directory that contains setup.py, setup.cfg, and versioneer.py . """ root = os.path.realpath(os.path.abspath(os.getcwd())) setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): # allow 'python path/to/setup.py COMMAND' root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): err = ("Versioneer was unable to run the project root directory. " "Versioneer requires setup.py to be executed from " "its immediate directory (like 'python setup.py COMMAND'), " "or in a way that lets it use sys.argv[0] to find the root " "(like 'python path/to/setup.py COMMAND').") raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools # tree) execute all dependencies in a single python process, so # "versioneer" may be imported multiple times, and python's shared # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. my_path = os.path.realpath(os.path.abspath(__file__)) me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): print("Warning: build in %s is using versioneer.py from %s" % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root def get_config_from_root(root): """Read the project setup.cfg file to determine Versioneer config.""" # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . root = Path(root) pyproject_toml = root / "pyproject.toml" setup_cfg = root / "setup.cfg" section = None if pyproject_toml.exists() and have_tomllib: try: with open(pyproject_toml, 'rb') as fobj: pp = tomllib.load(fobj) section = pp['tool']['versioneer'] except (tomllib.TOMLDecodeError, KeyError): pass if not section: parser = configparser.ConfigParser() with open(setup_cfg) as cfg_file: parser.read_file(cfg_file) parser.get("versioneer", "VCS") # raise error if missing section = parser["versioneer"] cfg = VersioneerConfig() cfg.VCS = section['VCS'] cfg.style = section.get("style", "") cfg.versionfile_source = section.get("versionfile_source") cfg.versionfile_build = section.get("versionfile_build") cfg.tag_prefix = section.get("tag_prefix") if cfg.tag_prefix in ("''", '""', None): cfg.tag_prefix = "" cfg.parentdir_prefix = section.get("parentdir_prefix") cfg.verbose = section.get("verbose") return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" # these dictionaries contain VCS-specific tools LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue if verbose: print("unable to run %s" % dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) return None, process.returncode return stdout, process.returncode LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. # Generated by versioneer-0.28 # https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" import errno import os import re import subprocess import sys from typing import Callable, Dict import functools def get_keywords(): """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: """Container for Versioneer configuration parameters.""" def get_config(): """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "%(STYLE)s" cfg.tag_prefix = "%(TAG_PREFIX)s" cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" cfg.verbose = False return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue if verbose: print("unable to run %%s" %% dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) return None, process.returncode return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: print("likely tags: %%s" %% ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %%s" %% r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%%s' doesn't start with prefix '%%s'" print(fmt %% (full_tag, tag_prefix)) pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" %% (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def plus_or_dot(pieces): """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces): """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver): """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces): """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%%d" %% (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%%s" %% pieces["short"] else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%%s" %% pieces["short"] return rendered def render_pep440_post_branch(pieces): """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%%s" %% pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%%s" %% pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces): """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces): """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%%s'" %% style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} def get_versions(): """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which # case we can only use expanded keywords. cfg = get_config() verbose = cfg.verbose try: return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass try: root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to find root of source tree", "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) return render(pieces, cfg.style) except NotThisMethod: pass try: if cfg.parentdir_prefix: return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) except NotThisMethod: pass return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} ''' @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def do_vcs_install(versionfile_source, ipy): """Git-specific installation logic for Versioneer. For Git, this means creating/changing .gitattributes to mark _version.py for export-subst keyword substitution. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] files = [versionfile_source] if ipy: files.append(ipy) if "VERSIONEER_PEP518" not in globals(): try: my_path = __file__ if my_path.endswith((".pyc", ".pyo")): my_path = os.path.splitext(my_path)[0] + ".py" versioneer_file = os.path.relpath(my_path) except NameError: versioneer_file = "versioneer.py" files.append(versioneer_file) present = False try: with open(".gitattributes", "r") as fobj: for line in fobj: if line.strip().startswith(versionfile_source): if "export-subst" in line.strip().split()[1:]: present = True break except OSError: pass if not present: with open(".gitattributes", "a+") as fobj: fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") SHORT_VERSION_PY = """ # This file was generated by 'versioneer.py' (0.28) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. import json version_json = ''' %s ''' # END VERSION_JSON def get_versions(): return json.loads(version_json) """ def versions_from_file(filename): """Try to determine the version from _version.py if present.""" try: with open(filename) as f: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) if not mo: mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) print("set %s to '%s'" % (filename, versions["version"])) def plus_or_dot(pieces): """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces): """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver): """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces): """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%d" % (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%d" % pieces["distance"] return rendered def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%s" % pieces["short"] return rendered def render_pep440_post_branch(pieces): """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces): """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces): """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} class VersioneerBadRootError(Exception): """The project root directory is unknown or missing key files.""" def get_versions(verbose=False): """Get the project version from whatever source is available. Returns dict with two keys: 'version' and 'full'. """ if "versioneer" in sys.modules: # see the discussion in cmdclass.py:get_cmdclass() del sys.modules["versioneer"] root = get_root() cfg = get_config_from_root(root) assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose assert cfg.versionfile_source is not None, \ "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) # extract version from first of: _version.py, VCS command (e.g. 'git # describe'), parentdir. This is meant to work for developers using a # source checkout, for users of a tarball created by 'setup.py sdist', # and for users of a tarball/zipball created by 'git archive' or github's # download-from-tag feature or the equivalent in other VCSes. get_keywords_f = handlers.get("get_keywords") from_keywords_f = handlers.get("keywords") if get_keywords_f and from_keywords_f: try: keywords = get_keywords_f(versionfile_abs) ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) if verbose: print("got version from expanded keyword %s" % ver) return ver except NotThisMethod: pass try: ver = versions_from_file(versionfile_abs) if verbose: print("got version from file %s %s" % (versionfile_abs, ver)) return ver except NotThisMethod: pass from_vcs_f = handlers.get("pieces_from_vcs") if from_vcs_f: try: pieces = from_vcs_f(cfg.tag_prefix, root, verbose) ver = render(pieces, cfg.style) if verbose: print("got version from VCS %s" % ver) return ver except NotThisMethod: pass try: if cfg.parentdir_prefix: ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) if verbose: print("got version from parentdir %s" % ver) return ver except NotThisMethod: pass if verbose: print("unable to compute version") return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} def get_version(): """Get the short version string for this project.""" return get_versions()["version"] def get_cmdclass(cmdclass=None): """Get the custom setuptools subclasses used by Versioneer. If the package uses a different cmdclass (e.g. one from numpy), it should be provide as an argument. """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and # 'easy_install .'), in which subdependencies of the main project are # built (using setup.py bdist_egg) in the same python process. Assume # a main project A and a dependency B, which use different versions # of Versioneer. A's setup.py imports A's Versioneer, leaving it in # sys.modules by the time B's setup.py is executed, causing B to run # with the wrong versioneer. Setuptools wraps the sub-dep builds in a # sandbox that restores sys.modules to it's pre-build state, so the # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. # Also see https://github.com/python-versioneer/python-versioneer/issues/52 cmds = {} if cmdclass is None else cmdclass.copy() # we add "version" to setuptools from setuptools import Command class cmd_version(Command): description = "report generated version string" user_options = [] boolean_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): vers = get_versions(verbose=True) print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) print(" dirty: %s" % vers.get("dirty")) print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) cmds["version"] = cmd_version # we override "build_py" in setuptools # # most invocation pathways end up running build_py: # distutils/build -> build_py # distutils/install -> distutils/build ->.. # setuptools/bdist_wheel -> distutils/install ->.. # setuptools/bdist_egg -> distutils/install_lib -> build_py # setuptools/install -> bdist_egg ->.. # setuptools/develop -> ? # pip install: # copies source tree to a tempdir before running egg_info/etc # if .git isn't copied too, 'git describe' will fail # then does setup.py bdist_wheel, or sometimes setup.py install # setup.py egg_info -> ? # pip install -e . and setuptool/editable_wheel will invoke build_py # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments if 'build_py' in cmds: _build_py = cmds['build_py'] else: from setuptools.command.build_py import build_py as _build_py class cmd_build_py(_build_py): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_py.run(self) if getattr(self, "editable_mode", False): # During editable installs `.py` and data files are # not copied to build_lib return # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py if 'build_ext' in cmds: _build_ext = cmds['build_ext'] else: from setuptools.command.build_ext import build_ext as _build_ext class cmd_build_ext(_build_ext): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_ext.run(self) if self.inplace: # build_ext --inplace will only build extensions in # build/lib<..> dir with no _version.py to write to. # As in place builds will already have a _version.py # in the module dir, we do not need to write one. return # now locate _version.py in the new build/ directory and replace # it with an updated value if not cfg.versionfile_build: return target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): print(f"Warning: {target_versionfile} does not exist, skipping " "version update. This can happen if you are running build_ext " "without first running build_py.") return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION # "product_version": versioneer.get_version(), # ... class cmd_build_exe(_build_exe): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() target_versionfile = cfg.versionfile_source print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) _build_exe.run(self) os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) cmds["build_exe"] = cmd_build_exe del cmds["build_py"] if 'py2exe' in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe except ImportError: from py2exe.distutils_buildexe import py2exe as _py2exe class cmd_py2exe(_py2exe): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() target_versionfile = cfg.versionfile_source print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) _py2exe.run(self) os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info if 'egg_info' in cmds: _egg_info = cmds['egg_info'] else: from setuptools.command.egg_info import egg_info as _egg_info class cmd_egg_info(_egg_info): def find_sources(self): # egg_info.find_sources builds the manifest list and writes it # in one shot super().find_sources() # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) self.filelist.append('versioneer.py') if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit self.filelist.append(cfg.versionfile_source) self.filelist.sort() self.filelist.remove_duplicates() # The write method is hidden in the manifest_maker instance that # generated the filelist and was thrown away # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') for f in self.filelist.files] manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') with open(manifest_filename, 'w') as fobj: fobj.write('\n'.join(normalized)) cmds['egg_info'] = cmd_egg_info # we override different "sdist" commands for both environments if 'sdist' in cmds: _sdist = cmds['sdist'] else: from setuptools.command.sdist import sdist as _sdist class cmd_sdist(_sdist): def run(self): versions = get_versions() self._versioneer_generated_versions = versions # unless we update this, the command will keep using the old # version self.distribution.metadata.version = versions["version"] return _sdist.run(self) def make_release_tree(self, base_dir, files): root = get_root() cfg = get_config_from_root(root) _sdist.make_release_tree(self, base_dir, files) # now locate _version.py in the new base_dir directory # (remembering that it may be a hardlink) and replace it with an # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, self._versioneer_generated_versions) cmds["sdist"] = cmd_sdist return cmds CONFIG_ERROR = """ setup.cfg is missing the necessary Versioneer configuration. You need a section like: [versioneer] VCS = git style = pep440 versionfile_source = src/myproject/_version.py versionfile_build = myproject/_version.py tag_prefix = parentdir_prefix = myproject- You will also need to edit your setup.py to use the results: import versioneer setup(version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), ...) Please read the docstring in ./versioneer.py for configuration instructions, edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. """ SAMPLE_CONFIG = """ # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the # resulting files. [versioneer] #VCS = git #style = pep440 #versionfile_source = #versionfile_build = #tag_prefix = #parentdir_prefix = """ OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ INIT_PY_SNIPPET = """ from . import {0} __version__ = {0}.get_versions()['version'] """ def do_setup(): """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) return 1 print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: old = f.read() except OSError: old = "" module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] snippet = INIT_PY_SNIPPET.format(module) if OLD_SNIPPET in old: print(" replacing boilerplate in %s" % ipy) with open(ipy, "w") as f: f.write(old.replace(OLD_SNIPPET, snippet)) elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: f.write(snippet) else: print(" %s unmodified" % ipy) else: print(" %s doesn't exist, ok" % ipy) ipy = None # Make VCS-specific changes. For git, this means creating/changing # .gitattributes to mark _version.py for export-subst keyword # substitution. do_vcs_install(cfg.versionfile_source, ipy) return 0 def scan_setup_py(): """Validate the contents of setup.py against Versioneer's expectations.""" found = set() setters = False errors = 0 with open("setup.py", "r") as f: for line in f.readlines(): if "import versioneer" in line: found.add("import") if "versioneer.get_cmdclass()" in line: found.add("cmdclass") if "versioneer.get_version()" in line: found.add("get_version") if "versioneer.VCS" in line: setters = True if "versioneer.versionfile_source" in line: setters = True if len(found) != 3: print("") print("Your setup.py appears to be missing some important items") print("(but I might be wrong). Please make sure it has something") print("roughly like the following:") print("") print(" import versioneer") print(" setup( version=versioneer.get_version(),") print(" cmdclass=versioneer.get_cmdclass(), ...)") print("") errors += 1 if setters: print("You should remove lines like 'versioneer.VCS = ' and") print("'versioneer.versionfile_source = ' . This configuration") print("now lives in setup.cfg, and should be removed from setup.py") print("") errors += 1 return errors def setup_command(): """Set up Versioneer and exit with appropriate error code.""" errors = do_setup() errors += scan_setup_py() sys.exit(1 if errors else 0) if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": setup_command()