pax_global_header00006660000000000000000000000064135377255340014530gustar00rootroot0000000000000052 comment=52960c10c6ad6d94e3f6defe88862f1f1345d091 pyannotate-1.2.0/000077500000000000000000000000001353772553400137125ustar00rootroot00000000000000pyannotate-1.2.0/.gitignore000066400000000000000000000000741353772553400157030ustar00rootroot00000000000000/.cache /*.egg-info /build /dist *.pyc __pycache__ /env* *~ pyannotate-1.2.0/.travis.yml000066400000000000000000000015071353772553400160260ustar00rootroot00000000000000language: python cache: pip matrix: include: - python: '2.7' - python: '3.4' - python: '3.5.1' # Special, it doesn't have typing.Text - python: '3.5' # Latest, e.g. 3.5.4 - python: '3.6' env: USE_MYPY=true - python: '3.7' dist: xenial # needed because Python 3.7 is broken on travis CI Trusty sudo: true env: USE_MYPY=true - python: '3.8-dev' dist: xenial # needed because Python 3.8 is broken on travis CI Trusty sudo: true env: USE_MYPY=true allow_failures: - python: 3.8-dev install: - pip install -r requirements.txt - if [[ $USE_MYPY == true ]]; then pip install -U mypy; fi script: - pytest - if [[ $USE_MYPY == true ]]; then mypy pyannotate_*; fi pyannotate-1.2.0/CONTRIBUTING.md000066400000000000000000000005551353772553400161500ustar00rootroot00000000000000First-time contributors: Please fill out the Dropbox Contributor License Agreement (CLA) at https://opensource.dropbox.com/cla/ (we currently check this manually, so we apologize for delays). Everyone: - Please run the tests (`pytest`) and make sure they pass. - Please add tests for the bug/feature you are fixing/adding. - Please follow PEP 8 for coding style. pyannotate-1.2.0/LICENSE000066400000000000000000000261241353772553400147240ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright (c) 2017 Dropbox, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pyannotate-1.2.0/MANIFEST.in000066400000000000000000000004241353772553400154500ustar00rootroot00000000000000graft example graft pyannotate_tools graft pyannotate_runtime include README.md include CONTRIBUTING.md include LICENSE include setup.py include setup.cfg include mypy.ini include requirements.txt global-exclude @* global-exclude *~ global-exclude *.pyc global-exclude *.json pyannotate-1.2.0/README.md000066400000000000000000000073551353772553400152030ustar00rootroot00000000000000PyAnnotate: Auto-generate PEP-484 annotations ============================================= Insert annotations into your source code based on call arguments and return types observed at runtime. For license and copyright see the end of this file. Blog post: http://mypy-lang.blogspot.com/2017/11/dropbox-releases-pyannotate-auto.html How to use ========== See also the example directory. Phase 1: Collecting types at runtime ------------------------------------ - Install the usual way (see "red tape" section below) - Add `from pyannotate_runtime import collect_types` to your test - Early in your test setup, call `collect_types.init_types_collection()` - Bracket your test execution between calls to `collect_types.start()` and `collect_types.stop()` (or use the context manager below) - When done, call `collect_types.dump_stats(filename)` All calls between the `start()` and `stop()` calls will be analyzed and the observed types will be written (in JSON form) to the filename you pass to `dump_stats()`. You can have multiple start/stop pairs per dump call. If you'd like to automatically collect types when you run `pytest`, see `example/example_conftest.py` and `example/README.md`. Instead of using `start()` and `stop()` you can also use a context manager: ``` collect_types.init_types_collection() with collect_types.collect(): collect_types.dump_stats() ``` Phase 2: Inserting types into your source code ---------------------------------------------- The command-line tool `pyannotate` can add annotations into your source code based on the annotations collected in phase 1. The key arguments are: - Use `--type-info FILE` to tell it the file you passed to `dump_stats()` - Positional arguments are source files you want to annotate - With no other flags the tool will print a diff indicating what it proposes to do but won't do anything. Review the output. - Add `-w` to make the tool actually update your files. (Use git or some other way to keep a backup.) At this point you should probably run mypy and iterate. You probably will have to tweak the changes to make mypy completely happy. Notes and tips -------------- - It's best to do one file at a time, at least until you're comfortable with the tool. - The tool doesn't touch functions that already have an annotation. - The tool can generate either of: - type comments, i.e. Python 2 style annotations - inline type annotations, i.e. Python 3 style annotations, using `--py3` in v1.0.7+ Red tape ======== Installation ------------ This should work for Python 2.7 as well as for Python 3.4 and higher. ``` pip install pyannotate ``` This installs several items: - A runtime module, pyannotate_runtime/collect_types.py, which collects and dumps types observed at runtime using a profiling hook. - A library package, pyannotate_tools, containing code that can read the data dumped by the runtime module and insert annotations into your source code. - An entry point, pyannotate, which runs the library package on your files. For dependencies, see setup.py and requirements.txt. Testing etc. ------------ To run the unit tests, use pytest: ``` pytest ``` TO DO ----- We'd love your help with some of these issues: - Better documentation. - Python 3 code generation. - Refactor the tool modules (currently its legacy architecture shines through). Acknowledgments --------------- The following people contributed significantly to this tool: - Tony Grue - Sergei Vorobev - Jukka Lehtosalo - Guido van Rossum Licence etc. ------------ 1. License: Apache 2.0. 2. Copyright attribution: Copyright (c) 2017 Dropbox, Inc. 3. External contributions to the project should be subject to Dropbox's Contributor License Agreement (CLA): https://opensource.dropbox.com/cla/ pyannotate-1.2.0/appveyor.yml000066400000000000000000000004111353772553400162760ustar00rootroot00000000000000environment: matrix: - PYTHON: "C:\\Python27" - PYTHON: "C:\\Python36-x64" build: off install: - "%PYTHON%\\python.exe -m pip install -r requirements.txt" - "%PYTHON%\\python.exe -m pip list" test_script: - "%PYTHON%\\python.exe -m pytest" pyannotate-1.2.0/example/000077500000000000000000000000001353772553400153455ustar00rootroot00000000000000pyannotate-1.2.0/example/.gitignore000066400000000000000000000000171353772553400173330ustar00rootroot00000000000000type_info.json pyannotate-1.2.0/example/README.md000066400000000000000000000026761353772553400166370ustar00rootroot00000000000000PyAnnotate example ================== To play with this example, first install PyAnnotate: ``` pip install pyannotate ``` Then run the driver.py file: ``` python driver.py ``` Expected contents of type_info.json (after running driver.py): ``` [ { "path": "gcd.py", "line": 1, "func_name": "main", "type_comments": [ "() -> None" ], "samples": 1 }, { "path": "gcd.py", "line": 5, "func_name": "gcd", "type_comments": [ "(int, int) -> int" ], "samples": 2 } ] ``` Now run the pyannotate tool, like this (note the -w flag -- without this it won't update the file): ``` pyannotate -w gcd.py ``` Expected output: ``` Refactored gcd.py --- gcd.py (original) +++ gcd.py (refactored) @@ -1,8 +1,10 @@ def main(): + # type: () -> None print(gcd(15, 10)) print(gcd(45, 12)) def gcd(a, b): + # type: (int, int) -> int while b: a, b = b, a%b return a Files that were modified: gcd.py ``` Alternative, using pytest ------------------------- For pytest users, the example_conftest.py file shows how to automatically configures pytest to collect types when running tests. The test_gcd.py file contains a simple test to demonstrate this. Copy the contents of example_conftest.py to your conftest.py file and run pytest; it will then generate a type_info.json file like the one above. pyannotate-1.2.0/example/driver.py000066400000000000000000000003501353772553400172100ustar00rootroot00000000000000from gcd import main from pyannotate_runtime import collect_types if __name__ == '__main__': collect_types.init_types_collection() with collect_types.collect(): main() collect_types.dump_stats('type_info.json') pyannotate-1.2.0/example/example_conftest.py000066400000000000000000000014501353772553400212570ustar00rootroot00000000000000# Configuration for pytest to automatically collect types. # Thanks to Guilherme Salgado. import pytest def pytest_collection_finish(session): """Handle the pytest collection finish hook: configure pyannotate. Explicitly delay importing `collect_types` until all tests have been collected. This gives gevent a chance to monkey patch the world before importing pyannotate. """ from pyannotate_runtime import collect_types collect_types.init_types_collection() @pytest.fixture(autouse=True) def collect_types_fixture(): from pyannotate_runtime import collect_types collect_types.start() yield collect_types.stop() def pytest_sessionfinish(session, exitstatus): from pyannotate_runtime import collect_types collect_types.dump_stats("type_info.json") pyannotate-1.2.0/example/gcd.py000066400000000000000000000001721353772553400164540ustar00rootroot00000000000000def main(): print(gcd(15, 10)) print(gcd(45, 12)) def gcd(a, b): while b: a, b = b, a%b return a pyannotate-1.2.0/example/test_gcd.py000066400000000000000000000001671353772553400175170ustar00rootroot00000000000000# Tests for gcd function. from gcd import gcd def test_gcd(): assert gcd(5, 10) == 5 assert gcd(12, 45) == 3 pyannotate-1.2.0/mypy.ini000066400000000000000000000000741353772553400154120ustar00rootroot00000000000000[mypy] ignore_missing_imports = True strict_optional = True pyannotate-1.2.0/pyannotate_runtime/000077500000000000000000000000001353772553400176375ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_runtime/__init__.py000066400000000000000000000000001353772553400217360ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_runtime/collect_types.py000066400000000000000000000764451353772553400231020ustar00rootroot00000000000000""" This module enables runtime type collection. Collected information can be used to automatically generate mypy annotation for the executed code paths. It uses python profiler callback to examine frames and record type info about arguments and return type. For the module consumer, the workflow looks like that: 1) call init_types_collection() from the main thread once 2) call start() to start the type collection 3) call stop() to stop the type collection 4) call dump_stats(file_name) to dump all collected info to the file as json You can repeat start() / stop() as many times as you want. The module is based on Tony's 2016 prototype D219371. """ from __future__ import ( absolute_import, division, print_function, ) import collections import inspect import json import opcode import os import sys import threading from inspect import ArgInfo from threading import Thread from mypy_extensions import TypedDict from six import iteritems from six.moves import range from six.moves.queue import Queue # type: ignore # No library stub yet from typing import ( Any, Callable, Dict, Iterable, Iterator, List, NamedTuple, Optional, Set, Sized, Tuple, TypeVar, Union, ) from contextlib import contextmanager MYPY=False if MYPY: # MYPY is True when mypy is running # 'Type' is only required for running mypy, not for running pyannotate from typing import Type # pylint: disable=invalid-name CO_GENERATOR = inspect.CO_GENERATOR # type: ignore def _my_hash(arg_list): # type: (List[Any]) -> int """Simple helper hash function""" res = 0 for arg in arg_list: res = res * 31 + hash(arg) return res # JSON object representing the collected data for a single function/method FunctionData = TypedDict('FunctionData', {'path': str, 'line': int, 'func_name': str, 'type_comments': List[str], 'samples': int}) class UnknownType(object): pass class NoReturnType(object): pass class TypeWasIncomparable(object): pass class FakeIterator(Iterable[Any], Sized): """ Container for iterator values. Note that FakeIterator([a, b, c]) is akin to list([a, b, c]); this is turned into IteratorType by resolve_type(). """ def __init__(self, values): # type: (List[Any]) -> None self.values = values def __iter__(self): # type: () -> Iterator[Any] for v in self.values: yield v def __len__(self): # type: () -> int return len(self.values) _NONE_TYPE = type(None) # type: Type[None] InternalType = Union['DictType', 'ListType', 'TupleType', 'SetType', 'IteratorType', 'type'] class DictType(object): """ Internal representation of Dict type. """ def __init__(self, key_type, val_type): # type: (TentativeType, TentativeType) -> None self.key_type = key_type self.val_type = val_type def __repr__(self): # type: () -> str if repr(self.key_type) == 'None': # We didn't see any values, so we don't know what's inside return 'Dict' else: return 'Dict[%s, %s]' % (repr(self.key_type), repr(self.val_type)) def __hash__(self): # type: () -> int return hash(self.key_type) if self.key_type else 0 def __eq__(self, other): # type: (object) -> bool if not isinstance(other, DictType): return False return self.val_type == other.val_type and self.key_type == other.key_type def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) class ListType(object): """ Internal representation of List type. """ def __init__(self, val_type): # type: (TentativeType) -> None self.val_type = val_type def __repr__(self): # type: () -> str if repr(self.val_type) == 'None': # We didn't see any values, so we don't know what's inside return 'List' else: return 'List[%s]' % (repr(self.val_type)) def __hash__(self): # type: () -> int return hash(self.val_type) if self.val_type else 0 def __eq__(self, other): # type: (object) -> bool if not isinstance(other, ListType): return False return self.val_type == other.val_type def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) class SetType(object): """ Internal representation of Set type. """ def __init__(self, val_type): # type: (TentativeType) -> None self.val_type = val_type def __repr__(self): # type: () -> str if repr(self.val_type) == 'None': # We didn't see any values, so we don't know what's inside return 'Set' else: return 'Set[%s]' % (repr(self.val_type)) def __hash__(self): # type: () -> int return hash(self.val_type) if self.val_type else 0 def __eq__(self, other): # type: (object) -> bool if not isinstance(other, SetType): return False return self.val_type == other.val_type def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) class IteratorType(object): """ Internal representation of Iterator type. """ def __init__(self, val_type): # type: (TentativeType) -> None self.val_type = val_type def __repr__(self): # type: () -> str if repr(self.val_type) == 'None': # We didn't see any values, so we don't know what's inside return 'Iterator' else: return 'Iterator[%s]' % (repr(self.val_type)) def __hash__(self): # type: () -> int return hash(self.val_type) if self.val_type else 0 def __eq__(self, other): # type: (object) -> bool if not isinstance(other, IteratorType): return False return self.val_type == other.val_type def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) class TupleType(object): """ Internal representation of Tuple type. """ def __init__(self, val_types): # type: (List[InternalType]) -> None self.val_types = val_types def __repr__(self): # type: () -> str return 'Tuple[%s]' % ', '.join([name_from_type(vt) for vt in self.val_types]) def __hash__(self): # type: () -> int return _my_hash(self.val_types) def __eq__(self, other): # type: (object) -> bool if not isinstance(other, TupleType): return False if len(self.val_types) != len(other.val_types): return False for i, v in enumerate(self.val_types): if v != other.val_types[i]: return False return True def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) class TentativeType(object): """ This class serves as internal representation of type for a type collection process. It can be merged with another instance of TentativeType to build up a broader sample. """ def __init__(self): # type: () -> None self.types_hashable = set() # type: Set[InternalType] self.types = [] # type: List[InternalType] def __hash__(self): # type: () -> int # These objects not immutable because there was a _large_ perf impact to being immutable. # Having a hash on a mutable object is dangerous, but is was much faster. # If you do change it, you need to # (a) pull it out of the set/table # (b) change it, # (c) stuff it back in return _my_hash([self.types, len(self.types_hashable)]) if self.types else 0 def __eq__(self, other): # type: (object) -> bool if not isinstance(other, TentativeType): return False if self.types_hashable != other.types_hashable: return False if len(self.types) != len(other.types): return False for i in self.types: if i not in other.types: return False return True def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) def add(self, type): # type: (InternalType) -> None """ Add type to the runtime type samples. """ try: if isinstance(type, SetType): if EMPTY_SET_TYPE in self.types_hashable: self.types_hashable.remove(EMPTY_SET_TYPE) elif isinstance(type, ListType): if EMPTY_LIST_TYPE in self.types_hashable: self.types_hashable.remove(EMPTY_LIST_TYPE) elif isinstance(type, IteratorType): if EMPTY_ITERATOR_TYPE in self.types_hashable: self.types_hashable.remove(EMPTY_ITERATOR_TYPE) elif isinstance(type, DictType): if EMPTY_DICT_TYPE in self.types_hashable: self.types_hashable.remove(EMPTY_DICT_TYPE) for item in self.types_hashable: if isinstance(item, DictType): if item.key_type == type.key_type: item.val_type.merge(type.val_type) return self.types_hashable.add(type) except (TypeError, AttributeError): try: if type not in self.types: self.types.append(type) except AttributeError: if TypeWasIncomparable not in self.types: self.types.append(TypeWasIncomparable) def merge(self, other): # type: (TentativeType) -> None """ Merge two TentativeType instances """ for hashables in other.types_hashable: self.add(hashables) for non_hashbles in other.types: self.add(non_hashbles) def __repr__(self): # type: () -> str if (len(self.types) + len(self.types_hashable) == 0) or ( len(self.types_hashable) == 1 and _NONE_TYPE in self.types_hashable): return 'None' else: type_format = '%s' filtered_types = self.types + [i for i in self.types_hashable if i != _NONE_TYPE] if _NONE_TYPE in self.types_hashable: type_format = 'Optional[%s]' if len(filtered_types) == 1: return type_format % name_from_type(filtered_types[0]) else: # use sorted() for predictable type order in the Union return type_format % ( 'Union[' + ', '.join(sorted([name_from_type(s) for s in filtered_types])) + ']') FunctionKey = NamedTuple('FunctionKey', [('path', str), ('line', int), ('func_name', str)]) # Inferred types for a function call ResolvedTypes = NamedTuple('ResolvedTypes', [('pos_args', List[InternalType]), ('varargs', Optional[List[InternalType]])]) # Task queue entry for calling a function with specific argument types KeyAndTypes = NamedTuple('KeyAndTypes', [('key', FunctionKey), ('types', ResolvedTypes)]) # Task queue entry for returning from a function with a value KeyAndReturn = NamedTuple('KeyAndReturn', [('key', FunctionKey), ('return_type', InternalType)]) # Combined argument and return types for a single function call Signature = NamedTuple('Signature', [('args', 'ArgTypes'), ('return_type', InternalType)]) BUILTIN_MODULES = {'__builtin__', 'builtins', 'exceptions'} def name_from_type(type_): # type: (InternalType) -> str """ Helper function to get PEP-484 compatible string representation of our internal types. """ if isinstance(type_, (DictType, ListType, TupleType, SetType, IteratorType)): return repr(type_) else: if type_.__name__ != 'NoneType': module = type_.__module__ if module in BUILTIN_MODULES or module == '': # Omit module prefix for known built-ins, for convenience. This # makes unit tests for this module simpler. # Also ignore '' modules so pyannotate can parse these types return type_.__name__ else: name = getattr(type_, '__qualname__', None) or type_.__name__ delim = '.' if '.' not in name else ':' return '%s%s%s' % (module, delim, name) else: return 'None' EMPTY_DICT_TYPE = DictType(TentativeType(), TentativeType()) EMPTY_LIST_TYPE = ListType(TentativeType()) EMPTY_SET_TYPE = SetType(TentativeType()) EMPTY_ITERATOR_TYPE = IteratorType(TentativeType()) # TODO: Make this faster def get_function_name_from_frame(frame): # type: (Any) -> str """ Heuristic to find the class-specified name by @guido For instance methods we return "ClassName.method_name" For functions we return "function_name" """ def bases_to_mro(cls, bases): # type: (type, List[type]) -> List[type] """ Convert __bases__ to __mro__ """ mro = [cls] for base in bases: if base not in mro: mro.append(base) sub_bases = getattr(base, '__bases__', None) if sub_bases: sub_bases = [sb for sb in sub_bases if sb not in mro and sb not in bases] if sub_bases: mro.extend(bases_to_mro(base, sub_bases)) return mro code = frame.f_code # This ought to be aggressively cached with the code object as key. funcname = code.co_name if code.co_varnames: varname = code.co_varnames[0] if varname == 'self': inst = frame.f_locals.get(varname) if inst is not None: try: mro = inst.__class__.__mro__ except AttributeError: mro = None try: bases = inst.__class__.__bases__ except AttributeError: bases = None else: mro = bases_to_mro(inst.__class__, bases) if mro: for cls in mro: bare_method = cls.__dict__.get(funcname) if bare_method and getattr(bare_method, '__code__', None) is code: return '%s.%s' % (cls.__name__, funcname) return funcname def resolve_type(arg): # type: (object) -> InternalType """ Resolve object to one of our internal collection types or generic built-in type. Args: arg: object to resolve """ arg_type = type(arg) if arg_type == list: assert isinstance(arg, list) # this line helps mypy figure out types sample = arg[:min(4, len(arg))] tentative_type = TentativeType() for sample_item in sample: tentative_type.add(resolve_type(sample_item)) return ListType(tentative_type) elif arg_type == set: assert isinstance(arg, set) # this line helps mypy figure out types sample = [] iterator = iter(arg) for i in range(0, min(4, len(arg))): sample.append(next(iterator)) tentative_type = TentativeType() for sample_item in sample: tentative_type.add(resolve_type(sample_item)) return SetType(tentative_type) elif arg_type == FakeIterator: assert isinstance(arg, FakeIterator) # this line helps mypy figure out types sample = [] iterator = iter(arg) for i in range(0, min(4, len(arg))): sample.append(next(iterator)) tentative_type = TentativeType() for sample_item in sample: tentative_type.add(resolve_type(sample_item)) return IteratorType(tentative_type) elif arg_type == tuple: assert isinstance(arg, tuple) # this line helps mypy figure out types sample = list(arg[:min(10, len(arg))]) return TupleType([resolve_type(sample_item) for sample_item in sample]) elif arg_type == dict: assert isinstance(arg, dict) # this line helps mypy figure out types key_tt = TentativeType() val_tt = TentativeType() for i, (k, v) in enumerate(iteritems(arg)): if i > 4: break key_tt.add(resolve_type(k)) val_tt.add(resolve_type(v)) return DictType(key_tt, val_tt) else: return type(arg) def prep_args(arg_info): # type: (ArgInfo) -> ResolvedTypes """ Resolve types from ArgInfo """ # pull out any varargs declarations filtered_args = [a for a in arg_info.args if getattr(arg_info, 'varargs', None) != a] # we don't care about self/cls first params (perhaps we can test if it's an instance/class method another way?) if filtered_args and (filtered_args[0] in ('self', 'cls')): filtered_args = filtered_args[1:] pos_args = [] # type: List[InternalType] if filtered_args: for arg in filtered_args: if isinstance(arg, str) and arg in arg_info.locals: # here we know that return type will be of type "type" resolved_type = resolve_type(arg_info.locals[arg]) pos_args.append(resolved_type) else: pos_args.append(type(UnknownType())) varargs = None # type: Optional[List[InternalType]] if arg_info.varargs: varargs_tuple = arg_info.locals[arg_info.varargs] # It's unclear what all the possible values for 'varargs_tuple' are, # so perform a defensive type check since we don't want to crash here. if isinstance(varargs_tuple, tuple): varargs = [resolve_type(arg) for arg in varargs_tuple[:4]] return ResolvedTypes(pos_args=pos_args, varargs=varargs) class ArgTypes(object): """ Internal representation of argument types in a single call """ def __init__(self, resolved_types): # type: (ResolvedTypes) -> None self.pos_args = [TentativeType() for _ in range(len(resolved_types.pos_args))] if resolved_types.pos_args: for i, arg in enumerate(resolved_types.pos_args): self.pos_args[i].add(arg) self.varargs = None # type: Optional[TentativeType] if resolved_types.varargs: self.varargs = TentativeType() for arg in resolved_types.varargs: self.varargs.add(arg) def __repr__(self): # type: () -> str return str({'pos_args': self.pos_args, 'varargs': self.varargs}) def __hash__(self): # type: () -> int return _my_hash(self.pos_args) + hash(self.varargs) def __eq__(self, other): # type: (object) -> bool return (isinstance(other, ArgTypes) and other.pos_args == self.pos_args and other.varargs == self.varargs) def __ne__(self, other): # type: (object) -> bool return not self.__eq__(other) # Collect at most this many type comments for each function. MAX_ITEMS_PER_FUNCTION = 8 # The most recent argument types collected for each function. Once we encounter # a corresponding return event, an item will be flushed and moved to # 'collected_comments'. collected_args = {} # type: Dict[FunctionKey, ArgTypes] # Collected unique type comments for each function, of form '(arg, ...) -> ret'. # There at most MAX_ITEMS_PER_FUNCTION items. collected_signatures = {} # type: Dict[FunctionKey, Set[Tuple[ArgTypes, InternalType]]] # Number of samples collected per function (we also count ones ignored after reaching # the maximum comment count per function). num_samples = {} # type: Dict[FunctionKey, int] def _make_type_comment(args_info, return_type): # type: (ArgTypes, InternalType) -> str """Generate a type comment of form '(arg, ...) -> ret'.""" if not args_info.pos_args: args_string = '' else: args_string = ', '.join([repr(t) for t in args_info.pos_args]) if args_info.varargs: varargs = '*%s' % repr(args_info.varargs) if args_string: args_string += ', %s' % varargs else: args_string = varargs return_name = name_from_type(return_type) return '(%s) -> %s' % (args_string, return_name) def _flush_signature(key, return_type): # type: (FunctionKey, InternalType) -> None """Store signature for a function. Assume that argument types have been stored previously to 'collected_args'. As the 'return_type' argument provides the return type, we now have a complete signature. As a side effect, removes the argument types for the function from 'collected_args'. """ signatures = collected_signatures.setdefault(key, set()) args_info = collected_args.pop(key) if len(signatures) < MAX_ITEMS_PER_FUNCTION: signatures.add((args_info, return_type)) num_samples[key] = num_samples.get(key, 0) + 1 def type_consumer(): # type: () -> None """ Infinite loop of the type consumer thread. It gets types to process from the task query. """ # we are not interested in profiling type_consumer itself # but we start it before any other thread while True: item = _task_queue.get() if isinstance(item, KeyAndTypes): if item.key in collected_args: # Previous call didn't get a corresponding return, perhaps because we # stopped collecting types in the middle of a call or because of # a recursive function. _flush_signature(item.key, UnknownType) collected_args[item.key] = ArgTypes(item.types) else: assert isinstance(item, KeyAndReturn) if item.key in collected_args: _flush_signature(item.key, item.return_type) _task_queue.task_done() _task_queue = Queue() # type: Queue[Union[KeyAndTypes, KeyAndReturn]] _consumer_thread = Thread(target=type_consumer) _consumer_thread.daemon = True _consumer_thread.start() running = False TOP_DIR = os.path.join(os.getcwd(), '') # current dir with trailing slash TOP_DIR_DOT = os.path.join(TOP_DIR, '.') TOP_DIR_LEN = len(TOP_DIR) def _make_sampling_sequence(n): # type: (int) -> List[int] """ Return a list containing the proposed call event sampling sequence. Return events are paired with call events and not counted separately. This is 0, 1, 2, ..., 4 plus 50, 100, 150, 200, etc. The total list size is n. """ seq = list(range(5)) i = 50 while len(seq) < n: seq.append(i) i += 50 return seq # We pre-compute the sampling sequence since 'x in ' is faster. MAX_SAMPLES_PER_FUNC = 500 sampling_sequence = frozenset(_make_sampling_sequence(MAX_SAMPLES_PER_FUNC)) LAST_SAMPLE = max(sampling_sequence) # Array of counters indexed by ID of code object. sampling_counters = {} # type: Dict[int, Optional[int]] # IDs of code objects for which the previous event was a call (awaiting return). call_pending = set() # type: Set[int] @contextmanager def collect(): # type: () -> Iterator[None] start() try: yield finally: stop() def pause(): # type: () -> None """ Deprecated, replaced by stop(). """ # In the future, do: warnings.warn("Function pause() has been replaced by start().", PendingDeprecationWarning) return stop() def stop(): # type: () -> None """ Start collecting type information. """ global running # pylint: disable=global-statement running = False _task_queue.join() def resume(): # type: () -> None """ Deprecated, replaced by start(). """ # In the future, do: warnings.warn("Function resume() has been replaced by stop().", PendingDeprecationWarning) return start() def start(): # type: () -> None """ Stop collecting type information. """ global running # pylint: disable=global-statement running = True sampling_counters.clear() def default_filter_filename(filename): # type: (Optional[str]) -> Optional[str] """Default filter for filenames. Returns either a normalized filename or None. You can pass your own filter to init_types_collection(). """ if filename is None: return None elif filename.startswith(TOP_DIR): if filename.startswith(TOP_DIR_DOT): # Skip subdirectories starting with dot (e.g. .vagrant). return None else: # Strip current directory and following slashes. return filename[TOP_DIR_LEN:].lstrip(os.sep) elif filename.startswith(os.sep): # Skip absolute paths not under current directory. return None else: return filename _filter_filename = default_filter_filename # type: Callable[[Optional[str]], Optional[str]] if sys.version_info[0] == 2: RETURN_VALUE_OPCODE = chr(opcode.opmap['RETURN_VALUE']) YIELD_VALUE_OPCODE = chr(opcode.opmap['YIELD_VALUE']) else: RETURN_VALUE_OPCODE = opcode.opmap['RETURN_VALUE'] YIELD_VALUE_OPCODE = opcode.opmap['YIELD_VALUE'] def _trace_dispatch(frame, event, arg): # type: (Any, str, Optional[Any]) -> None """ This is the main hook passed to setprofile(). It implement python profiler interface. Arguments are described in https://docs.python.org/2/library/sys.html#sys.settrace """ # Bail if we're not tracing. if not running: return # Get counter for this code object. Bail if we don't care about this function. # An explicit None is stored in the table when we no longer care. code = frame.f_code key = id(code) n = sampling_counters.get(key, 0) if n is None: return if event == 'call': # Bump counter and bail depending on sampling sequence. sampling_counters[key] = n + 1 # Each function gets traced at most MAX_SAMPLES_PER_FUNC times per run. # NOTE: There's a race condition if two threads call the same function. # I don't think we should care, so what if it gets probed an extra time. if n not in sampling_sequence: if n > LAST_SAMPLE: sampling_counters[key] = None # We're no longer interested in this function. call_pending.discard(key) # Avoid getting events out of sync return # Mark that we are looking for a return from this code object. call_pending.add(key) elif event == 'return': if key not in call_pending: # No pending call event -- ignore this event. We only collect # return events when we know the corresponding call event. return call_pending.discard(key) # Avoid race conditions else: # Ignore other events, such as c_call and c_return. return # Track calls under current directory only. filename = _filter_filename(code.co_filename) if filename: func_name = get_function_name_from_frame(frame) if not func_name or func_name[0] == '<': # Could be a lambda or a comprehension; we're not interested. sampling_counters[key] = None else: function_key = FunctionKey(filename, code.co_firstlineno, func_name) if event == 'call': # TODO(guido): Make this faster arg_info = inspect.getargvalues(frame) # type: ArgInfo resolved_types = prep_args(arg_info) _task_queue.put(KeyAndTypes(function_key, resolved_types)) elif event == 'return': # This event is also triggered if a function yields or raises an exception. # We can tell the difference by looking at the bytecode. # (We don't get here for C functions so the bytecode always exists.) last_opcode = code.co_code[frame.f_lasti] if last_opcode == RETURN_VALUE_OPCODE: if code.co_flags & CO_GENERATOR: # Return from a generator. t = resolve_type(FakeIterator([])) else: t = resolve_type(arg) elif last_opcode == YIELD_VALUE_OPCODE: # Yield from a generator. # TODO: Unify generators -- currently each YIELD is turned into # a separate call, so a function yielding ints and strs will be # typed as Union[Iterator[int], Iterator[str]] -- this should be # Iterator[Union[int, str]]. t = resolve_type(FakeIterator([arg])) else: # This branch is also taken when returning from a generator. # TODO: returning non-trivial values from generators, per PEP 380; # and async def / await stuff. t = NoReturnType _task_queue.put(KeyAndReturn(function_key, t)) else: sampling_counters[key] = None # We're not interested in this function. T = TypeVar('T') def _filter_types(types_dict): # type: (Dict[FunctionKey, T]) -> Dict[FunctionKey, T] """Filter type info before dumping it to the file.""" def exclude(k): # type: (FunctionKey) -> bool """Exclude filter""" return k.path.startswith('<') or k.func_name == '' return {k: v for k, v in iteritems(types_dict) if not exclude(k)} def _dump_impl(): # type: () -> List[FunctionData] """Internal implementation for dump_stats and dumps_stats""" filtered_signatures = _filter_types(collected_signatures) sorted_by_file = sorted(iteritems(filtered_signatures), key=(lambda p: (p[0].path, p[0].line, p[0].func_name))) res = [] # type: List[FunctionData] for function_key, signatures in sorted_by_file: comments = [_make_type_comment(args, ret_type) for args, ret_type in signatures] res.append( { 'path': function_key.path, 'line': function_key.line, 'func_name': function_key.func_name, 'type_comments': comments, 'samples': num_samples.get(function_key, 0), } ) return res def dump_stats(filename): # type: (str) -> None """ Write collected information to file. Args: filename: absolute filename """ res = _dump_impl() f = open(filename, 'w') json.dump(res, f, indent=4) f.close() def dumps_stats(): # type: () -> str """ Return collected information as a json string. """ res = _dump_impl() return json.dumps(res, indent=4) def init_types_collection(filter_filename=default_filter_filename): # type: (Callable[[Optional[str]], Optional[str]]) -> None """ Setup profiler hooks to enable type collection. Call this one time from the main thread. The optional argument is a filter that maps a filename (from code.co_filename) to either a normalized filename or None. For the default filter see default_filter_filename(). """ global _filter_filename _filter_filename = filter_filename sys.setprofile(_trace_dispatch) threading.setprofile(_trace_dispatch) def stop_types_collection(): # type: () -> None """ Remove profiler hooks. """ sys.setprofile(None) threading.setprofile(None) # type: ignore pyannotate-1.2.0/pyannotate_runtime/tests/000077500000000000000000000000001353772553400210015ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_runtime/tests/__init__.py000066400000000000000000000000001353772553400231000ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_runtime/tests/test_collect_types.py000066400000000000000000000505761353772553400253000ustar00rootroot00000000000000"""Tests for collect_types""" from __future__ import ( absolute_import, division, print_function, ) import contextlib import json import os import sched import sys import time import unittest from collections import namedtuple from threading import Thread from six import PY2 from typing import ( Any, Dict, Iterator, List, Optional, Tuple, Union, ) try: from typing import Text except ImportError: # In Python 3.5.1 stdlib, typing.py does not define Text Text = str # type: ignore from pyannotate_runtime import collect_types # A bunch of random functions and classes to test out type collection # Disable a whole bunch of lint warnings for simplicity # pylint:disable=invalid-name # pylint:disable=blacklisted-name # pylint:disable=missing-docstring FooNamedTuple = namedtuple('FooNamedTuple', 'foo bar') def print_int(i): # type: (Any) -> Any print(i) def noop_dec(a): # type: (Any) -> Any return a def discard(a): # type: (Any) -> None pass @noop_dec class FoosParent(object): pass class FooObject(FoosParent): class FooNested(object): pass class FooReturn(FoosParent): pass class WorkerClass(object): def __init__(self, special_num, foo): # type: (Any, Any) -> None self._special_num = special_num self._foo = foo @noop_dec def do_work(self, i, haz): # type: (Any, Any) -> Any print_int(i) return EOFError() @classmethod def do_work_clsmthd(cls, i, haz=None): # type: (Any, Any) -> Any print_int(i) return EOFError() class EventfulHappenings(object): def __init__(self): # type: () -> None self.handlers = [] # type: Any def add_handler(self, handler): # type: (Any) -> Any self.handlers.append(handler) def something_happened(self, a, b): # type: (Any, Any) -> Any for h in self.handlers: h(a, b) return 1999 # A class that is old style under python 2 class OldStyleClass: def foo(self, x): # type: (Any) -> Any return x def i_care_about_whats_happening(y, z): # type: (Any, Any) -> Any print_int(y) print(z) return FooReturn() def takes_different_lists(l): # type: (Any) -> Any pass def takes_int_lists(l): # type: (Any) -> Any pass def takes_int_float_lists(l): # type: (Any) -> Any pass def takes_int_to_str_dict(d): # type: (Any) -> Any pass def takes_int_to_multiple_val_dict(d): # type: (Any) -> Any pass def recursive_dict(d): # type: (Any) -> Any pass def empty_then_not_dict(d): # type: (Any) -> Any return d def empty_then_not_list(l): # type: (Any) -> Any pass def tuple_verify(t): # type: (Any) -> Any return t def problematic_dup(uni, bol): # type: (Text, bool) -> Tuple[Dict[Text, Union[List, int, Text]],bytes] return {u"foo": [], u"bart": u'ads', u"bax": 23}, b'str' def two_dict_comprehensions(): # type: () -> Dict[int, Dict[Tuple[int, int], int]] d = {1: {1: 2}} return { i: { (i, k): l for k, l in j.items() } for i, j in d.items() } class TestBaseClass(unittest.TestCase): def setUp(self): # type: () -> None super(TestBaseClass, self).setUp() # Stats in the same format as the generated JSON. self.stats = [] # type: List[collect_types.FunctionData] def tearDown(self): # type: () -> None collect_types.stop_types_collection() def load_stats(self): # type: () -> None self.stats = json.loads(collect_types.dumps_stats()) @contextlib.contextmanager def collecting_types(self): # type: () -> Iterator[None] collect_types.collected_args = {} collect_types.collected_signatures = {} collect_types.num_samples = {} collect_types.sampling_counters = {} collect_types.call_pending = set() collect_types.start() yield None collect_types.stop() self.load_stats() def assert_type_comments(self, func_name, comments): # type: (str, List[str]) -> None """Assert that we generated expected comment for the func_name function in self.stats""" stat_items = [item for item in self.stats if item.get('func_name') == func_name] if not comments and not stat_items: # If we expect no comments, it's okay if nothing was collected. return assert len(stat_items) == 1 item = stat_items[0] if set(item['type_comments']) != set(comments): print('Actual:') for comment in sorted(item['type_comments']): print(' ' + comment) print('Expected:') for comment in sorted(comments): print(' ' + comment) assert set(item['type_comments']) == set(comments) assert len(item['type_comments']) == len(comments) assert os.path.join(collect_types.TOP_DIR, item['path']) == __file__ class TestCollectTypes(TestBaseClass): def setUp(self): # type: () -> None super(TestCollectTypes, self).setUp() collect_types.init_types_collection() # following type annotations are intentionally use Any, # because we are testing runtime type collection def foo(self, int_arg, list_arg): # type: (Any, Any) -> None """foo""" self.bar(int_arg, list_arg) def bar(self, int_arg, list_arg): # type: (Any, Any) -> Any """bar""" return len(self.baz(list_arg)) + int_arg def baz(self, list_arg): # type: (Any) -> Any """baz""" return set([int(s) for s in list_arg]) def test_type_collection_on_main_thread(self): # type: () -> None with self.collecting_types(): self.foo(2, ['1', '2']) self.assert_type_comments('TestCollectTypes.foo', ['(int, List[str]) -> None']) self.assert_type_comments('TestCollectTypes.bar', ['(int, List[str]) -> int']) self.assert_type_comments('TestCollectTypes.baz', ['(List[str]) -> Set[int]']) def bar_another_thread(self, int_arg, list_arg): # type: (Any, Any) -> Any """bar""" return len(self.baz_another_thread(list_arg)) + int_arg def baz_another_thread(self, list_arg): # type: (Any) -> Any """baz""" return set([int(s) for s in list_arg]) def test_type_collection_on_another_thread(self): # type: () -> None with self.collecting_types(): t = Thread(target=self.bar_another_thread, args=(100, ['1', '2', '3'],)) t.start() t.join() self.assert_type_comments('TestCollectTypes.baz_another_thread', ['(List[str]) -> Set[int]']) def test_run_a_bunch_of_tests(self): # type: () -> None with self.collecting_types(): to = FooObject() wc = WorkerClass(42, to) s = sched.scheduler(time.time, time.sleep) event_source = EventfulHappenings() s.enter(.001, 1, wc.do_work, ([52, 'foo,', 32], FooNamedTuple('ab', 97))) s.enter(.002, 1, wc.do_work, ([52, 32], FooNamedTuple('bc', 98))) s.enter(.003, 1, wc.do_work_clsmthd, (52, FooNamedTuple('de', 99))) s.enter(.004, 1, event_source.add_handler, (i_care_about_whats_happening,)) s.enter(.005, 1, event_source.add_handler, (lambda a, b: print_int(a),)) s.enter(.006, 1, event_source.something_happened, (1, 'tada')) s.run() takes_different_lists([42, 'as', 323, 'a']) takes_int_lists([42, 323, 3231]) takes_int_float_lists([42, 323.2132, 3231]) takes_int_to_str_dict({2: 'a', 4: 'd'}) takes_int_to_multiple_val_dict({3: 'a', 4: None, 5: 232}) recursive_dict({3: {3: 'd'}, 4: {3: 'd'}}) empty_then_not_dict({}) empty_then_not_dict({3: {3: 'd'}, 4: {3: 'd'}}) empty_then_not_list([]) empty_then_not_list([1, 2]) empty_then_not_list([1, 2]) tuple_verify((1, '4')) tuple_verify((1, '4')) problematic_dup(u'ha', False) problematic_dup(u'ha', False) OldStyleClass().foo(10) discard(FooObject.FooNested()) # TODO(svorobev): add checks for the rest of the functions # print_int, self.assert_type_comments( 'WorkerClass.__init__', ['(int, pyannotate_runtime.tests.test_collect_types.FooObject) -> None']) self.assert_type_comments( 'do_work_clsmthd', ['(int, pyannotate_runtime.tests.test_collect_types.FooNamedTuple) -> EOFError']) self.assert_type_comments('OldStyleClass.foo', ['(int) -> int']) # Need __qualname__ to get this right if sys.version_info >= (3, 3): self.assert_type_comments( 'discard', ['(pyannotate_runtime.tests.test_collect_types:FooObject.FooNested) -> None']) # TODO: that could be better self.assert_type_comments('takes_different_lists', ['(List[Union[int, str]]) -> None']) # TODO: that should work # self.assert_type_comment('empty_then_not_dict', # '(Dict[int, Dict[int, str]]) -> Dict[int, Dict[int, str]]') self.assert_type_comments('empty_then_not_list', ['(List[int]) -> None', '(List) -> None']) if PY2: self.assert_type_comments( 'problematic_dup', ['(unicode, bool) -> Tuple[Dict[unicode, Union[List, int, unicode]], str]']) else: self.assert_type_comments( 'problematic_dup', ['(str, bool) -> Tuple[Dict[str, Union[List, int, str]], bytes]']) def test_two_signatures(self): # type: () -> None def identity(x): # type: (Any) -> Any return x with self.collecting_types(): identity(1) identity('x') self.assert_type_comments('identity', ['(int) -> int', '(str) -> str']) def test_many_signatures(self): # type: () -> None def identity2(x): # type: (Any) -> Any return x with self.collecting_types(): for x in 1, 'x', 2, 'y', slice(1), 1.1, None, False, bytearray(), (), [], set(): for _ in range(50): identity2(x) # We collect at most 8 distinct signatures. self.assert_type_comments('identity2', ['(int) -> int', '(str) -> str', '(slice) -> slice', '(float) -> float', '(None) -> None', '(bool) -> bool', '(bytearray) -> bytearray', '(Tuple[]) -> Tuple[]']) def test_default_args(self): # type: () -> None def func_default(x=0, y=None): # type: (Any, Any) -> Any return x with self.collecting_types(): func_default() func_default('') func_default(1.1, True) self.assert_type_comments('func_default', ['(int, None) -> int', '(str, None) -> str', '(float, bool) -> float']) def test_keyword_args(self): # type: () -> None def func_kw(x, y): # type: (Any, Any) -> Any return x with self.collecting_types(): func_kw(y=1, x='') func_kw(**{'x': 1.1, 'y': None}) self.assert_type_comments('func_kw', ['(str, int) -> str', '(float, None) -> float']) def test_no_return(self): # type: () -> None def func_always_fail(x): # type: (Any) -> Any raise ValueError def func_sometimes_fail(x): # type: (Any) -> Any if x == 0: raise RuntimeError return x with self.collecting_types(): try: func_always_fail(1) except Exception: pass try: func_always_fail('') except Exception: pass try: func_always_fail(1) except Exception: pass try: func_sometimes_fail(0) except Exception: pass func_sometimes_fail('') try: func_sometimes_fail(0) except Exception: pass self.assert_type_comments('func_always_fail', ['(int) -> pyannotate_runtime.collect_types.NoReturnType', '(str) -> pyannotate_runtime.collect_types.NoReturnType']) self.assert_type_comments('func_sometimes_fail', ['(int) -> pyannotate_runtime.collect_types.NoReturnType', '(str) -> str']) def test_only_return(self): # type: () -> None def only_return(x): # type: (int) -> str collect_types.start() return '' only_return(1) collect_types.stop() self.load_stats() # No entry is stored if we only have a return event with no matching call. self.assert_type_comments('only_return', []) def test_callee_star_args(self): # type: () -> None def callee_star_args(x, *y): # type: (Any, *Any) -> Any return 0 with self.collecting_types(): callee_star_args(0) callee_star_args(1, '') callee_star_args(slice(1), 1.1, True) callee_star_args(*(False, 1.1, '')) self.assert_type_comments('callee_star_args', ['(int) -> int', '(int, *str) -> int', '(slice, *Union[bool, float]) -> int', '(bool, *Union[float, str]) -> int']) def test_caller_star_args(self): # type: () -> None def caller_star_args(x, y=None): # type: (Any, Any) -> Any return 0 with self.collecting_types(): caller_star_args(*(1,)) caller_star_args(*('', 1.1)) self.assert_type_comments('caller_star_args', ['(int, None) -> int', '(str, float) -> int']) def test_star_star_args(self): # type: () -> None def star_star_args(x, **kw): # type: (Any, **Any) -> Any return 0 with self.collecting_types(): star_star_args(1, y='', z=True) star_star_args(**{'x': True, 'a': 1.1}) self.assert_type_comments('star_star_args', ['(int) -> int', '(bool) -> int']) def test_fully_qualified_type_name_with_sub_package(self): # type: () -> None def identity_qualified(x): # type: (Any) -> Any return x with self.collecting_types(): identity_qualified(collect_types.TentativeType()) self.assert_type_comments( 'identity_qualified', ['(pyannotate_runtime.collect_types.TentativeType) -> ' 'pyannotate_runtime.collect_types.TentativeType']) def test_recursive_function(self): # type: () -> None def recurse(x): # type: (Any) -> Any if len(x) == 0: return 1.1 else: recurse(x[1:]) return x[0] with self.collecting_types(): recurse((1, '', True)) self.assert_type_comments( 'recurse', ['(Tuple[]) -> float', '(Tuple[bool]) -> pyannotate_runtime.collect_types.UnknownType', '(Tuple[str, bool]) -> pyannotate_runtime.collect_types.UnknownType', '(Tuple[int, str, bool]) -> pyannotate_runtime.collect_types.UnknownType']) def test_recursive_function_2(self): # type: () -> None def recurse(x): # type: (Any) -> Any if x == 0: recurse('') recurse(1.1) return False else: return x with self.collecting_types(): # The return event for the initial call is mismatched because of # the recursive calls, so we'll have to drop the return type. recurse(0) self.assert_type_comments( 'recurse', ['(str) -> str', '(float) -> float', '(int) -> pyannotate_runtime.collect_types.UnknownType']) def test_ignoring_c_calls(self): # type: () -> None def func(x): # type: (Any) -> Any a = [1] # Each of these generates a c_call/c_return event pair. y = len(a), len(a), len(a), len(a), len(a), len(a), len(a), len(a), len(a), len(a) y = len(a), len(a), len(a), len(a), len(a), len(a), len(a), len(a), len(a), len(a) str(y) return x with self.collecting_types(): func(1) func('') self.assert_type_comments('func', ['(int) -> int', '(str) -> str']) def test_no_crash_on_nested_dict_comps(self): # type: () -> None with self.collecting_types(): two_dict_comprehensions() self.assert_type_comments('two_dict_comprehensions', ['() -> Dict[int, Dict[Tuple[int, int], int]]']) def test_skip_lambda(self): # type: () -> None with self.collecting_types(): (lambda: None)() (lambda x: x)(0) (lambda x, y: x+y)(0, 0) assert self.stats == [] def test_unknown_module_types(self): # type: () -> None def func_with_unknown_module_types(c): # type: (Any) -> Any return c with self.collecting_types(): ns = { '__name__': '' } # type: Dict[str, Any] exec('class C(object): pass', ns) func_with_unknown_module_types(ns['C']()) self.assert_type_comments('func_with_unknown_module_types', ['(C) -> C']) def test_yield_basic(self): # type: () -> None def gen(n, a): for i in range(n): yield a with self.collecting_types(): list(gen(10, 'x')) self.assert_type_comments('gen', ['(int, str) -> Iterator[str]']) def test_yield_various(self): # type: () -> None def gen(n, a, b): for i in range(n): yield a yield b with self.collecting_types(): list(gen(10, 'x', 1)) list(gen(0, 0, 0)) # TODO: This should really return Iterator[Union[int, str]] self.assert_type_comments('gen', ['(int, str, int) -> Iterator[int]', '(int, str, int) -> Iterator[str]']) def test_yield_empty(self): # type: () -> None def gen(): if False: yield with self.collecting_types(): list(gen()) self.assert_type_comments('gen', ['() -> Iterator']) def foo(arg): # type: (Any) -> Any return [arg] class TestInitWithFilter(TestBaseClass): def always_foo(self, filename): # type: (Optional[str]) -> Optional[str] return 'foo.py' def always_none(self, filename): # type: (Optional[str]) -> Optional[str] return None def test_init_with_filter(self): # type: () -> None collect_types.init_types_collection(self.always_foo) with self.collecting_types(): foo(42) assert len(self.stats) == 1 assert self.stats[0]['path'] == 'foo.py' def test_init_with_none_filter(self): # type: () -> None collect_types.init_types_collection(self.always_none) with self.collecting_types(): foo(42) assert self.stats == [] pyannotate-1.2.0/pyannotate_tools/000077500000000000000000000000001353772553400173145ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/__init__.py000066400000000000000000000000001353772553400214130ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/annotations/000077500000000000000000000000001353772553400216515ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/annotations/__init__.py000066400000000000000000000000171353772553400237600ustar00rootroot00000000000000# type: ignore pyannotate-1.2.0/pyannotate_tools/annotations/__main__.py000066400000000000000000000136171353772553400237530ustar00rootroot00000000000000from __future__ import print_function import argparse import json import logging import os import sys from lib2to3.main import StdoutRefactoringTool from typing import Any, Dict, List, Optional from pyannotate_tools.annotations.main import generate_annotations_json_string, unify_type_comments from pyannotate_tools.fixes.fix_annotate_json import FixAnnotateJson parser = argparse.ArgumentParser() parser.add_argument('--type-info', default='type_info.json', metavar="FILE", help="JSON input file (default type_info.json)") parser.add_argument('--uses-signature', action='store_true', help="JSON input uses a signature format") parser.add_argument('-p', '--print-function', action='store_true', help="Assume print is a function") parser.add_argument('-w', '--write', action='store_true', help="Write output files") parser.add_argument('-j', '--processes', type=int, default=1, metavar="N", help="Use N parallel processes (default no parallelism)") parser.add_argument('-v', '--verbose', action='store_true', help="More verbose output") parser.add_argument('-q', '--quiet', action='store_true', help="Don't show diffs") parser.add_argument('-d', '--dump', action='store_true', help="Dump raw type annotations (filter by files, default all)") parser.add_argument('-a', '--auto-any', action='store_true', help="Annotate everything with 'Any', without reading type_info.json") parser.add_argument('files', nargs='*', metavar="FILE", help="Files and directories to update with annotations") parser.add_argument('-s', '--only-simple', action='store_true', help="Only annotate functions with trivial types") parser.add_argument('--python-version', action='store', default='2', help="Choose annotation style, 2 for Python 2 with comments (the " "default), 3 for Python 3 with annotation syntax" ) parser.add_argument('--py2', '-2', action='store_const', dest='python_version', const='2', help="Annotate for Python 2 with comments (default)") parser.add_argument('--py3', '-3', action='store_const', dest='python_version', const='3', help="Annotate for Python 3 with argument and return value annotations") class ModifiedRefactoringTool(StdoutRefactoringTool): """Class that gives a nicer error message for bad encodings.""" def refactor_file(self, filename, write=False, doctests_only=False): try: super(ModifiedRefactoringTool, self).refactor_file( filename, write=write, doctests_only=doctests_only) except SyntaxError as err: if str(err).startswith("unknown encoding:"): self.log_error("Can't parse %s: %s", filename, err) else: raise def dump_annotations(type_info, files): """Dump annotations out of type_info, filtered by files. If files is non-empty, only dump items either if the path in the item matches one of the files exactly, or else if one of the files is a path prefix of the path. """ with open(type_info) as f: data = json.load(f) for item in data: path, line, func_name = item['path'], item['line'], item['func_name'] if files and path not in files: for f in files: if path.startswith(os.path.join(f, '')): break else: continue # Outer loop print("%s:%d: in %s:" % (path, line, func_name)) type_comments = item['type_comments'] signature = unify_type_comments(type_comments) arg_types = signature['arg_types'] return_type = signature['return_type'] print(" # type: (%s) -> %s" % (", ".join(arg_types), return_type)) def main(args_override=None): # type: (Optional[List[str]]) -> None # Parse command line. args = parser.parse_args(args_override) if not args.files and not args.dump: parser.error("At least one file/directory is required") if args.python_version not in ('2', '3'): sys.exit('--python-version must be 2 or 3') annotation_style = 'py' + args.python_version # Set up logging handler. level = logging.DEBUG if args.verbose else logging.INFO logging.basicConfig(format='%(message)s', level=level) if args.dump: dump_annotations(args.type_info, args.files) return if args.auto_any: fixers = ['pyannotate_tools.fixes.fix_annotate'] else: # Produce nice error message if type_info.json not found. try: with open(args.type_info) as f: contents = f.read() except IOError as err: sys.exit("Can't open type info file: %s" % err) # Run pass 2 with output into a variable. if args.uses_signature: data = json.loads(contents) # type: List[Any] else: data = generate_annotations_json_string( args.type_info, only_simple=args.only_simple) # Run pass 3 with input from that variable. FixAnnotateJson.init_stub_json_from_data(data, args.files[0]) fixers = ['pyannotate_tools.fixes.fix_annotate_json'] flags = {'print_function': args.print_function, 'annotation_style': annotation_style} rt = ModifiedRefactoringTool( fixers=fixers, options=flags, explicit=fixers, nobackups=True, show_diffs=not args.quiet) if not rt.errors: rt.refactor(args.files, write=args.write, num_processes=args.processes) if args.processes == 1: rt.summarize() else: logging.info("(In multi-process per-file warnings are lost)") if not args.write: logging.info("NOTE: this was a dry run; use -w to write files") if __name__ == '__main__': main() pyannotate-1.2.0/pyannotate_tools/annotations/infer.py000066400000000000000000000205121353772553400233260ustar00rootroot00000000000000"""Infer an annotation from a set of concrete runtime type signatures. The main entry point is 'infer_annotation'. """ from typing import Dict, Iterable, List, Optional, Set, Tuple from pyannotate_tools.annotations.parse import parse_type_comment from pyannotate_tools.annotations.types import ( AbstractType, AnyType, ARG_POS, Argument, ClassType, is_optional, TupleType, UnionType, NoReturnType, ) IGNORED_ITEMS = { 'unittest.mock.Mock', 'unittest.mock.MagicMock', 'mock.mock.Mock', 'mock.mock.MagicMock', } class InferError(Exception): """Raised if we can't infer a signature for some reason.""" def infer_annotation(type_comments): # type: (List[str]) -> Tuple[List[Argument], AbstractType] """Given some type comments, return a single inferred signature. Args: type_comments: Strings of form '(arg1, ... argN) -> ret' Returns: Tuple of (argument types and kinds, return type). """ assert type_comments args = {} # type: Dict[int, Set[Argument]] returns = set() for comment in type_comments: arg_types, return_type = parse_type_comment(comment) for i, arg_type in enumerate(arg_types): args.setdefault(i, set()).add(arg_type) returns.add(return_type) combined_args = [] for i in sorted(args): arg_infos = list(args[i]) kind = argument_kind(arg_infos) if kind is None: raise InferError('Ambiguous argument kinds:\n' + '\n'.join(type_comments)) types = [arg.type for arg in arg_infos] combined = combine_types(types) if str(combined) == 'None': # It's very rare for an argument to actually be typed `None`, more likely than # not we simply don't have any data points for this argument. combined = UnionType([ClassType('None'), AnyType()]) if kind != ARG_POS and (len(str(combined)) > 120 or isinstance(combined, UnionType)): # Avoid some noise. combined = AnyType() combined_args.append(Argument(combined, kind)) combined_return = combine_types(returns) return combined_args, combined_return def argument_kind(args): # type: (List[Argument]) -> Optional[str] """Return the kind of an argument, based on one or more descriptions of the argument. Return None if every item does not have the same kind. """ kinds = set(arg.kind for arg in args) if len(kinds) != 1: return None return kinds.pop() def combine_types(types): # type: (Iterable[AbstractType]) -> AbstractType """Given some types, return a combined and simplified type. For example, if given 'int' and 'List[int]', return Union[int, List[int]]. If given 'int' and 'int', return just 'int'. """ items = simplify_types(types) if len(items) == 1: return items[0] else: return UnionType(items) def simplify_types(types): # type: (Iterable[AbstractType]) -> List[AbstractType] """Given some types, give simplified types representing the union of types.""" flattened = flatten_types(types) items = filter_ignored_items(flattened) items = [simplify_recursive(item) for item in items] items = merge_items(items) items = dedupe_types(items) # We have to remove reundant items after everything has been simplified and # merged as this simplification may be what makes items redundant. items = remove_redundant_items(items) if len(items) > 3: return [AnyType()] else: return items def simplify_recursive(typ): # type: (AbstractType) -> AbstractType """Simplify all components of a type.""" if isinstance(typ, UnionType): return combine_types(typ.items) elif isinstance(typ, ClassType): simplified = ClassType(typ.name, [simplify_recursive(arg) for arg in typ.args]) args = simplified.args if (simplified.name == 'Dict' and len(args) == 2 and isinstance(args[0], ClassType) and args[0].name in ('str', 'Text') and isinstance(args[1], UnionType) and not is_optional(args[1])): # Looks like a potential case for TypedDict, which we don't properly support yet. return ClassType('Dict', [args[0], AnyType()]) return simplified elif isinstance(typ, TupleType): return TupleType([simplify_recursive(item) for item in typ.items]) return typ def flatten_types(types): # type: (Iterable[AbstractType]) -> List[AbstractType] flattened = [] for item in types: if not isinstance(item, UnionType): flattened.append(item) else: flattened.extend(flatten_types(item.items)) return flattened def dedupe_types(types): # type: (Iterable[AbstractType]) -> List[AbstractType] return sorted(set(types), key=lambda t: str(t)) def filter_ignored_items(items): # type: (List[AbstractType]) -> List[AbstractType] result = [item for item in items if not isinstance(item, ClassType) or item.name not in IGNORED_ITEMS] return result or [AnyType()] def remove_redundant_items(items): # type: (List[AbstractType]) -> List[AbstractType] """Filter out redundant union items.""" result = [] for item in items: for other in items: if item is not other and is_redundant_union_item(item, other): break else: result.append(item) return result def is_redundant_union_item(first, other): # type: (AbstractType, AbstractType) -> bool """If union has both items, is the first one redundant? For example, if first is 'str' and the other is 'Text', return True. If items are equal, return False. """ if isinstance(first, ClassType) and isinstance(other, ClassType): if first.name == 'str' and other.name == 'Text': return True elif first.name == 'bool' and other.name == 'int': return True elif first.name == 'int' and other.name == 'float': return True elif (first.name in ('List', 'Dict', 'Set') and other.name == first.name): if not first.args and other.args: return True elif len(first.args) == len(other.args) and first.args: result = all(first_arg == other_arg or other_arg == AnyType() for first_arg, other_arg in zip(first.args, other.args)) return result return False def merge_items(items): # type: (List[AbstractType]) -> List[AbstractType] """Merge union items that can be merged.""" result = [] while items: item = items.pop() merged = None for i, other in enumerate(items): merged = merged_type(item, other) if merged: break if merged: del items[i] items.append(merged) else: result.append(item) return list(reversed(result)) def merged_type(t, s): # type: (AbstractType, AbstractType) -> Optional[AbstractType] """Return merged type if two items can be merged in to a different, more general type. Return None if merging is not possible. """ if isinstance(t, TupleType) and isinstance(s, TupleType): if len(t.items) == len(s.items): return TupleType([combine_types([ti, si]) for ti, si in zip(t.items, s.items)]) all_items = t.items + s.items if all_items and all(item == all_items[0] for item in all_items[1:]): # Merge multiple compatible fixed-length tuples into a variable-length tuple type. return ClassType('Tuple', [all_items[0]]) elif (isinstance(t, TupleType) and isinstance(s, ClassType) and s.name == 'Tuple' and len(s.args) == 1): if all(item == s.args[0] for item in t.items): # Merge fixed-length tuple and variable-length tuple. return s elif isinstance(s, TupleType) and isinstance(t, ClassType) and t.name == 'Tuple': return merged_type(s, t) elif isinstance(s, NoReturnType): return t elif isinstance(t, NoReturnType): return s elif isinstance(s, AnyType): # This seems to be usually desirable, since Anys tend to come from unknown types. return t elif isinstance(t, AnyType): # Similar to above. return s return None pyannotate-1.2.0/pyannotate_tools/annotations/main.py000066400000000000000000000054151353772553400231540ustar00rootroot00000000000000"""Main entry point to mypy annotation inference utility.""" import json from typing import List from mypy_extensions import TypedDict from pyannotate_tools.annotations.types import ARG_STAR, ARG_STARSTAR from pyannotate_tools.annotations.infer import infer_annotation from pyannotate_tools.annotations.parse import parse_json # Schema of a function signature in the output Signature = TypedDict('Signature', {'arg_types': List[str], 'return_type': str}) # Schema of a function in the output FunctionData = TypedDict('FunctionData', {'path': str, 'line': int, 'func_name': str, 'signature': Signature, 'samples': int}) SIMPLE_TYPES = {'None', 'int', 'float', 'str', 'bytes', 'bool'} def unify_type_comments(type_comments): # type: (List[str]) -> Signature arg_types, return_type = infer_annotation(type_comments) arg_strs = [] for arg, kind in arg_types: arg_str = str(arg) if kind == ARG_STAR: arg_str = '*%s' % arg_str elif kind == ARG_STARSTAR: arg_str = '**%s' % arg_str arg_strs.append(arg_str) return { 'arg_types': arg_strs, 'return_type': str(return_type), } def is_signature_simple(signature): # type: (Signature) -> bool return (all(x.lstrip('*') in SIMPLE_TYPES for x in signature['arg_types']) and signature['return_type'] in SIMPLE_TYPES) def generate_annotations_json_string(source_path, only_simple=False): # type: (str, bool) -> List[FunctionData] """Produce annotation data JSON file from a JSON file with runtime-collected types. Data formats: * The source JSON is a list of pyannotate_tools.annotations.parse.RawEntry items. * The output JSON is a list of FunctionData items. """ items = parse_json(source_path) results = [] for item in items: signature = unify_type_comments(item.type_comments) if is_signature_simple(signature) or not only_simple: data = { 'path': item.path, 'line': item.line, 'func_name': item.func_name, 'signature': signature, 'samples': item.samples } # type: FunctionData results.append(data) return results def generate_annotations_json(source_path, target_path, only_simple=False): # type: (str, str, bool) -> None """Like generate_annotations_json_string() but writes JSON to a file.""" results = generate_annotations_json_string(source_path, only_simple=only_simple) with open(target_path, 'w') as f: json.dump(results, f, sort_keys=True, indent=4) pyannotate-1.2.0/pyannotate_tools/annotations/parse.py000066400000000000000000000246531353772553400233470ustar00rootroot00000000000000"""Parse type annotations collected at runtime by collect_types and dumped as JSON. Parse JSON data and also parse type comment strings into type objects. The collect_types tool is in pyannotate_runtime/collect_types.py. """ import json import re import sys from typing import Any, List, Mapping, Set, Tuple try: from typing import Text except ImportError: # In Python 3.5.1 stdlib, typing.py does not define Text Text = str # type: ignore from mypy_extensions import NoReturn, TypedDict from pyannotate_tools.annotations.types import ( AbstractType, AnyType, ARG_POS, ARG_STAR, ARG_STARSTAR, Argument, ClassType, TupleType, UnionType, NoReturnType, ) PY2 = sys.version_info < (3,) # Rules for replacing some type names that aren't valid Python names or that # are otherwise invalid. TYPE_FIXUPS = { # The dictionary-* names come from Python 2 `__class__.__name__` values # from `dict.iterkeys()`, etc. Python 3 uses valid names. 'dictionary-keyiterator': 'Iterator', 'dictionary-valueiterator': 'Iterator', 'dictionary-itemiterator': 'Iterator', 'pyannotate_runtime.collect_types.UnknownType': 'Any', 'pyannotate_runtime.collect_types.NoReturnType': 'mypy_extensions.NoReturn', 'function': 'Callable', 'functools.partial': 'Callable', 'long': 'int', 'unicode': 'Text', 'generator': 'Iterator', 'listiterator': 'Iterator', 'instancemethod': 'Callable', 'itertools.imap': 'Iterator', 'operator.methodcaller': 'Callable', 'method': 'Callable', 'method-wrapper': 'Callable', 'mappingproxy': 'Mapping', 'file': 'IO[bytes]', 'instance': 'Any', 'collections.defaultdict': 'Dict', } # Input JSON data entry RawEntry = TypedDict('RawEntry', {'path': Text, 'line': int, 'func_name': Text, 'type_comments': List[Text], 'samples': int}) class FunctionInfo(object): """Deserialized raw runtime information for a single function (based on RawEntry)""" def __init__(self, path, line, func_name, type_comments, samples): # type: (str, int, str, List[str], int) -> None self.path = path self.line = line self.func_name = func_name self.type_comments = type_comments self.samples = samples class ParseError(Exception): """Raised on any type comment parse error. The 'comment' attribute contains the comment that produced the error. """ def __init__(self, comment): # type: (str) -> None super(ParseError, self).__init__('Invalid type comment: %s' % comment) self.comment = comment def parse_json(path): # type: (str) -> List[FunctionInfo] """Deserialize a JSON file containing runtime collected types. The input JSON is expected to to have a list of RawEntry items. """ with open(path) as f: data = json.load(f) # type: List[RawEntry] result = [] def assert_type(value, typ): # type: (object, type) -> None assert isinstance(value, typ), '%s: Unexpected type %r' % (path, type(value).__name__) def assert_dict_item(dictionary, key, typ): # type: (Mapping[Any, Any], str, type) -> None assert key in dictionary, '%s: Missing dictionary key %r' % (path, key) value = dictionary[key] assert isinstance(value, typ), '%s: Unexpected type %r for key %r' % ( path, type(value).__name__, key) assert_type(data, list) for item in data: assert_type(item, dict) assert_dict_item(item, 'path', Text) assert_dict_item(item, 'line', int) assert_dict_item(item, 'func_name', Text) assert_dict_item(item, 'type_comments', list) for comment in item['type_comments']: assert_type(comment, Text) assert_type(item['samples'], int) info = FunctionInfo(encode(item['path']), item['line'], encode(item['func_name']), [encode(comment) for comment in item['type_comments']], item['samples']) result.append(info) return result class Token(object): """Abstract base class for tokens used for parsing type comments""" text = '' class DottedName(Token): """An identifier token, such as 'List', 'int' or 'package.name'""" def __init__(self, text): # type: (str) -> None self.text = text def __repr__(self): # type: () -> str return 'DottedName(%s)' % self.text class Separator(Token): """A separator or punctuator token such as '(', '[' or '->'""" def __init__(self, text): # type: (str) -> None self.text = text def __repr__(self): # type: () -> str return self.text class End(Token): """A token representing the end of a type comment""" def __repr__(self): # type: () -> str return 'End()' def tokenize(s): # type: (str) -> List[Token] """Translate a type comment into a list of tokens.""" original = s tokens = [] # type: List[Token] while True: if not s: tokens.append(End()) return tokens elif s[0] == ' ': s = s[1:] elif s[0] in '()[],*': tokens.append(Separator(s[0])) s = s[1:] elif s[:2] == '->': tokens.append(Separator('->')) s = s[2:] else: m = re.match(r'[-\w]+(\s*(\.|:)\s*[-/\w]*)*', s) if not m: raise ParseError(original) fullname = m.group(0) fullname = fullname.replace(' ', '') if fullname in TYPE_FIXUPS: fullname = TYPE_FIXUPS[fullname] # pytz creates classes with the name of the timezone being used: # https://github.com/stub42/pytz/blob/f55399cddbef67c56db1b83e0939ecc1e276cf42/src/pytz/tzfile.py#L120-L123 # This causes pyannotates to crash as it's invalid to have a class # name with a `/` in it (e.g. "pytz.tzfile.America/Los_Angeles") if fullname.startswith('pytz.tzfile.'): fullname = 'datetime.tzinfo' if '-' in fullname or '/' in fullname: # Not a valid Python name; there are many places that # generate these, so we just substitute Any rather # than crashing. fullname = 'Any' tokens.append(DottedName(fullname)) s = s[len(m.group(0)):] def parse_type_comment(comment): # type: (str) -> Tuple[List[Argument], AbstractType] """Parse a type comment of form '(arg1, ..., argN) -> ret'.""" return Parser(comment).parse() class Parser(object): """Implementation of the type comment parser""" def __init__(self, comment): # type: (str) -> None self.comment = comment self.tokens = tokenize(comment) self.i = 0 def parse(self): # type: () -> Tuple[List[Argument], AbstractType] self.expect('(') arg_types = [] # type: List[Argument] stars_seen = set() # type: Set[str] while self.lookup() != ')': if self.lookup() == '*': self.expect('*') if self.lookup() == '*': if '**' in stars_seen: self.fail() self.expect('*') star_star = True else: if stars_seen: self.fail() star_star = False arg_type = self.parse_type() if star_star: arg_types.append(Argument(arg_type, ARG_STARSTAR)) stars_seen.add('**') else: arg_types.append(Argument(arg_type, ARG_STAR)) stars_seen.add('*') else: if stars_seen: self.fail() arg_type = self.parse_type() arg_types.append(Argument(arg_type, ARG_POS)) if self.lookup() == ',': self.expect(',') elif self.lookup() == ')': break self.expect(')') self.expect('->') ret_type = self.parse_type() if not isinstance(self.next(), End): self.fail() return arg_types, ret_type def parse_type_list(self): # type: () -> List[AbstractType] types = [] while self.lookup() not in (')', ']'): typ = self.parse_type() types.append(typ) if self.lookup() == ',': self.expect(',') elif self.lookup() not in (')', ']'): self.fail() return types def parse_type(self): # type: () -> AbstractType t = self.next() if not isinstance(t, DottedName): self.fail() if t.text == 'Any': return AnyType() elif t.text == 'mypy_extensions.NoReturn': return NoReturnType() elif t.text == 'Tuple': self.expect('[') args = self.parse_type_list() self.expect(']') return TupleType(args) elif t.text == 'Union': self.expect('[') items = self.parse_type_list() self.expect(']') if len(items) == 1: return items[0] elif len(items) == 0: self.fail() else: return UnionType(items) else: if self.lookup() == '[': self.expect('[') args = self.parse_type_list() self.expect(']') if t.text == 'Optional' and len(args) == 1: return UnionType([args[0], ClassType('None')]) return ClassType(t.text, args) else: return ClassType(t.text) def expect(self, s): # type: (str) -> None if self.tokens[self.i].text != s: self.fail() self.i += 1 def lookup(self): # type: () -> str return self.tokens[self.i].text def next(self): # type: () -> Token token = self.tokens[self.i] self.i += 1 return token def fail(self): # type: () -> NoReturn raise ParseError(self.comment) def encode(s): # type: (Text) -> str if PY2: return s.encode('ascii') else: return s pyannotate-1.2.0/pyannotate_tools/annotations/tests/000077500000000000000000000000001353772553400230135ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/annotations/tests/__init__.py000066400000000000000000000000171353772553400251220ustar00rootroot00000000000000# type: ignore pyannotate-1.2.0/pyannotate_tools/annotations/tests/dundermain_test.py000066400000000000000000000077421353772553400265640ustar00rootroot00000000000000"""Some (nearly) end-to-end testing.""" import json import os import re import shutil import sys import tempfile import unittest # There seems to be no way to have this work and type-check without an # explicit version check. :-( if sys.version_info[0] == 2: from cStringIO import StringIO else: from io import StringIO from typing import Iterator, List from pyannotate_tools.annotations.__main__ import main as dunder_main class TestDunderMain(unittest.TestCase): def setUp(self): # type: () -> None self.tempdirname = tempfile.mkdtemp() self.tempfiles = [] # type: List[str] self.olddir = os.getcwd() os.chdir(self.tempdirname) def tearDown(self): # type: () -> None os.chdir(self.olddir) shutil.rmtree(self.tempdirname) def write_file(self, name, data): # type: (str, str) -> None self.tempfiles.append(name) with open(name, 'w') as f: f.write(data) def test_help(self): # type: () -> None self.main_test(["--help"], r"^usage:", r"^$", 0) def test_preview(self): # type: () -> None self.prototype_test(write=False) def test_final(self): # type: () -> None self.prototype_test(write=True) with open('gcd.py') as f: lines = [line.strip() for line in f.readlines()] assert '# type: (int, int) -> int' in lines def test_bad_encoding_message(self): # type: () -> None source_text = "# coding: unknownencoding\ndef f():\n pass\n" self.write_file('gcd.py', source_text) self.write_file('type_info.json', '[]') encoding_message = "Can't parse gcd.py: unknown encoding: unknownencoding" self.main_test(['gcd.py'], r'\A\Z', r'\A' + re.escape(encoding_message), 0) def prototype_test(self, write): # type: (bool) -> None type_info = [ { "path": "gcd.py", "line": 1, "func_name": "gcd", "type_comments": [ "(int, int) -> int" ], "samples": 2 } ] source_text = """\ def gcd(a, b): while b: a, b = b, a%b return a """ stdout_expected = """\ --- gcd.py (original) +++ gcd.py (refactored) @@ -1,4 +1,5 @@ def gcd(a, b): + # type: (int, int) -> int while b: a, b = b, a%b return a """ if not write: stderr_expected = """\ Refactored gcd.py Files that need to be modified: gcd.py NOTE: this was a dry run; use -w to write files """ else: stderr_expected = """\ Refactored gcd.py Files that were modified: gcd.py """ self.write_file('type_info.json', json.dumps(type_info)) self.write_file('gcd.py', source_text) args = ['gcd.py'] if write: args.append('-w') self.main_test(args, re.escape(stdout_expected) + r'\Z', re.escape(stderr_expected) + r'\Z', 0) def main_test(self, args, stdout_pattern, stderr_pattern, exit_code): # type: (List[str], str, str, int) -> None save_stdout = sys.stdout save_stderr = sys.stderr stdout = StringIO() stderr = StringIO() try: sys.stdout = stdout sys.stderr = stderr dunder_main(args) code = 0 except SystemExit as err: code = err.code finally: sys.stdout = save_stdout sys.stderr = save_stderr stdout_value = stdout.getvalue() stderr_value = stderr.getvalue() assert re.match(stdout_pattern, stdout_value) match = re.match(stderr_pattern, stderr_value) ## if not match: print("\nNah") ## else: print("\nYa!") ## print(stderr_value) ## import pdb; pdb.set_trace() assert code == exit_code pyannotate-1.2.0/pyannotate_tools/annotations/tests/infer_test.py000066400000000000000000000220471353772553400255340ustar00rootroot00000000000000import unittest from typing import List, Tuple from pyannotate_tools.annotations.infer import ( flatten_types, infer_annotation, merge_items, remove_redundant_items, ) from pyannotate_tools.annotations.types import ( AbstractType, AnyType, ARG_POS, ARG_STAR, ClassType, TupleType, UnionType, NoReturnType, ) class TestInfer(unittest.TestCase): def test_simple(self): # type: () -> None self.assert_infer(['(int) -> str'], ([(ClassType('int'), ARG_POS)], ClassType('str'))) def test_infer_union_arg(self): # type: () -> None self.assert_infer(['(int) -> None', '(str) -> None'], ([(UnionType([ClassType('int'), ClassType('str')]), ARG_POS)], ClassType('None'))) def test_infer_union_return(self): # type: () -> None self.assert_infer(['() -> int', '() -> str'], ([], UnionType([ClassType('int'), ClassType('str')]))) def test_star_arg(self): # type: () -> None self.assert_infer(['(int) -> None', '(int, *bool) -> None'], ([(ClassType('int'), ARG_POS), (ClassType('bool'), ARG_STAR)], ClassType('None'))) def test_merge_unions(self): # type: () -> None self.assert_infer(['(Union[int, str]) -> None', '(Union[str, None]) -> None'], ([(UnionType([ClassType('int'), ClassType('str'), ClassType('None')]), ARG_POS)], ClassType('None'))) def test_remove_redundant_union_item(self): # type: () -> None self.assert_infer(['(str) -> None', '(unicode) -> None'], ([(ClassType('Text'), ARG_POS)], ClassType('None'))) def test_remove_redundant_dict_item(self): # type: () -> None self.assert_infer(['(Dict[str, Any]) -> None', '(Dict[str, str]) -> None'], ([(ClassType('Dict', [ClassType('str'), AnyType()]), ARG_POS)], ClassType('None'))) def test_remove_redundant_dict_item_when_simplified(self): # type: () -> None self.assert_infer(['(Dict[str, Any]) -> None', '(Dict[str, Union[str, List, Dict, int]]) -> None'], ([(ClassType('Dict', [ClassType('str'), AnyType()]), ARG_POS)], ClassType('None'))) def test_simplify_list_item_types(self): # type: () -> None self.assert_infer(['(List[Union[bool, int]]) -> None'], ([(ClassType('List', [ClassType('int')]), ARG_POS)], ClassType('None'))) def test_simplify_potential_typed_dict(self): # type: () -> None # Fall back to Dict[x, Any] in case of a complex Dict type. self.assert_infer(['(Dict[str, Union[int, str]]) -> Any'], ([(ClassType('Dict', [ClassType('str'), AnyType()]), ARG_POS)], AnyType())) self.assert_infer(['(Dict[Text, Union[int, str]]) -> Any'], ([(ClassType('Dict', [ClassType('Text'), AnyType()]), ARG_POS)], AnyType())) # Not a potential TypedDict so ordinary simplification applies. self.assert_infer(['(Dict[str, Union[str, Text]]) -> Any'], ([(ClassType('Dict', [ClassType('str'), ClassType('Text')]), ARG_POS)], AnyType())) self.assert_infer(['(Dict[str, Union[int, None]]) -> Any'], ([(ClassType('Dict', [ClassType('str'), UnionType([ClassType('int'), ClassType('None')])]), ARG_POS)], AnyType())) def test_simplify_multiple_empty_collections(self): # type: () -> None self.assert_infer(['() -> Tuple[List, List[x]]', '() -> Tuple[List, List]'], ([], TupleType([ClassType('List'), ClassType('List', [ClassType('x')])]))) def assert_infer(self, comments, expected): # type: (List[str], Tuple[List[Tuple[AbstractType, str]], AbstractType]) -> None actual = infer_annotation(comments) assert actual == expected def test_infer_ignore_mock(self): # type: () -> None self.assert_infer(['(mock.mock.Mock) -> None', '(str) -> None'], ([(ClassType('str'), ARG_POS)], ClassType('None'))) def test_infer_ignore_mock_fallback_to_any(self): # type: () -> None self.assert_infer(['(mock.mock.Mock) -> str', '(mock.mock.Mock) -> int'], ([(AnyType(), ARG_POS)], UnionType([ClassType('str'), ClassType('int')]))) def test_infer_none_argument(self): # type: () -> None self.assert_infer(['(None) -> None'], ([(UnionType([ClassType('None'), AnyType()]), ARG_POS)], ClassType('None'))) CT = ClassType class TestRedundantItems(unittest.TestCase): def test_cannot_simplify(self): # type: () -> None for first, second in ((CT('str'), CT('int')), (CT('List', [CT('int')]), CT('List', [CT('str')])), (CT('List'), CT('Set', [CT('int')]))): assert remove_redundant_items([first, second]) == [first, second] assert remove_redundant_items([second, first]) == [second, first] def test_simplify_simple(self): # type: () -> None for first, second in ((CT('str'), CT('Text')), (CT('bool'), CT('int')), (CT('int'), CT('float'))): assert remove_redundant_items([first, second]) == [second] assert remove_redundant_items([second, first]) == [second] def test_simplify_multiple(self): # type: () -> None assert remove_redundant_items([CT('Text'), CT('str'), CT('bool'), CT('int'), CT('X')]) == [CT('Text'), CT('int'), CT('X')] def test_simplify_generics(self): # type: () -> None for first, second in ((CT('List'), CT('List', [CT('Text')])), (CT('Set'), CT('Set', [CT('Text')])), (CT('Dict'), CT('Dict', [CT('str'), CT('int')]))): assert remove_redundant_items([first, second]) == [second] class TestMergeUnionItems(unittest.TestCase): def test_cannot_merge(self): # type: () -> None for first, second in ((CT('str'), CT('Text')), (CT('List', [CT('int')]), CT('List', [CT('str')]))): assert merge_items([first, second]) == [first, second] assert merge_items([second, first]) == [second, first] assert merge_items([first, second, first]) == [first, second, first] def test_merge_union_of_same_length_tuples(self): # type: () -> None assert merge_items([TupleType([CT('str')]), TupleType([CT('int')])]) == [TupleType([UnionType([CT('str'), CT('int')])])] assert merge_items([TupleType([CT('str')]), TupleType([CT('Text')])]) == [TupleType([CT('Text')])] def test_merge_tuples_with_different_lengths(self): # type: () -> None assert merge_items([ TupleType([CT('str')]), TupleType([CT('str'), CT('str')])]) == [CT('Tuple', [CT('str')])] assert merge_items([ TupleType([]), TupleType([CT('str')]), TupleType([CT('str'), CT('str')])]) == [CT('Tuple', [CT('str')])] # Don't merge if types aren't identical assert merge_items([ TupleType([CT('str')]), TupleType([CT('str'), CT('int')])]) == [TupleType([CT('str')]), TupleType([CT('str'), CT('int')])] def test_merge_union_containing_no_return(self): # type: () -> None assert merge_items([CT('int'), NoReturnType()]) == [CT('int')] assert merge_items([NoReturnType(), CT('int')]) == [CT('int')] class TestFlattenTypes(unittest.TestCase): def test_nested_tuples(self): # type: () -> None assert flatten_types([UnionType([UnionType([CT('int'), CT('str')]), CT('X')])]) == [ CT('int'), CT('str'), CT('X')] pyannotate-1.2.0/pyannotate_tools/annotations/tests/main_test.py000066400000000000000000000125771353772553400253640ustar00rootroot00000000000000import contextlib import os import tempfile import textwrap import unittest from typing import Iterator from pyannotate_tools.annotations.infer import InferError from pyannotate_tools.annotations.main import (generate_annotations_json, generate_annotations_json_string) class TestMain(unittest.TestCase): def test_generation(self): # type: () -> None data = """ [ { "path": "pkg/thing.py", "line": 422, "func_name": "my_function", "type_comments": [ "(List[int], str) -> None" ], "samples": 3 } ] """ with self.temporary_file() as target_path: with self.temporary_json_file(data) as source_path: generate_annotations_json(source_path, target_path) with open(target_path) as target: actual = target.read() actual = actual.replace(' \n', '\n') expected = textwrap.dedent("""\ [ { "func_name": "my_function", "line": 422, "path": "pkg/thing.py", "samples": 3, "signature": { "arg_types": [ "List[int]", "str" ], "return_type": "None" } } ]""") assert actual == expected def test_ambiguous_kind(self): # type: () -> None data = """ [ { "path": "pkg/thing.py", "line": 422, "func_name": "my_function", "type_comments": [ "(List[int], str) -> None", "(List[int], *str) -> None" ], "samples": 3 } ] """ with self.assertRaises(InferError) as e: with self.temporary_json_file(data) as source_path: generate_annotations_json(source_path, '/dummy') assert str(e.exception) == textwrap.dedent("""\ Ambiguous argument kinds: (List[int], str) -> None (List[int], *str) -> None""") def test_generate_to_memory(self): # type: () -> None data = """ [ { "path": "pkg/thing.py", "line": 422, "func_name": "my_function", "type_comments": [ "(List[int], str) -> None" ], "samples": 3 } ] """ with self.temporary_json_file(data) as source_path: output_data = generate_annotations_json_string(source_path) assert output_data == [ { "path": "pkg/thing.py", "line": 422, "func_name": "my_function", "signature": { "arg_types": [ "List[int]", "str" ], "return_type": "None" }, "samples": 3 } ] with self.temporary_json_file(data) as source_path: output_data = generate_annotations_json_string(source_path, only_simple=True) assert output_data == [] def test_generate_simple_signatures(self): # type: () -> None data = """ [ { "path": "pkg/thing.py", "line": 422, "func_name": "complex_function", "type_comments": [ "(List[int], str) -> None" ], "samples": 3 }, { "path": "pkg/thing.py", "line": 9000, "func_name": "simple_function", "type_comments": [ "(int, str) -> None" ], "samples": 3 } ] """ with self.temporary_json_file(data) as source_path: output_data = generate_annotations_json_string(source_path, only_simple=True) assert output_data == [ { "path": "pkg/thing.py", "line": 9000, "func_name": "simple_function", "signature": { "arg_types": [ "int", "str" ], "return_type": "None" }, "samples": 3 } ] @contextlib.contextmanager def temporary_json_file(self, data): # type: (str) -> Iterator[str] source = None try: with tempfile.NamedTemporaryFile(mode='w', delete=False) as source: source.write(data) yield source.name finally: if source is not None: os.remove(source.name) @contextlib.contextmanager def temporary_file(self): # type: () -> Iterator[str] target = None try: with tempfile.NamedTemporaryFile(mode='w', delete=False) as target: pass yield target.name finally: if target is not None: os.remove(target.name) pyannotate-1.2.0/pyannotate_tools/annotations/tests/parse_test.py000066400000000000000000000165211353772553400255430ustar00rootroot00000000000000import os import tempfile import unittest from typing import List, Optional, Tuple from pyannotate_tools.annotations.parse import parse_json, parse_type_comment, ParseError, tokenize from pyannotate_tools.annotations.types import ( AbstractType, AnyType, ARG_POS, ARG_STAR, ARG_STARSTAR, Argument, ClassType, TupleType, UnionType, NoReturnType, ) class TestParseError(unittest.TestCase): def test_str_conversion(self): # type: () -> None assert str(ParseError('(int -> str')) == 'Invalid type comment: (int -> str' class TestParseJson(unittest.TestCase): def test_parse_json(self): # type: () -> None data = """ [ { "path": "pkg/thing.py", "line": 422, "func_name": "my_function", "type_comments": [ "(int) -> None", "(str) -> None" ], "samples": 3 } ] """ f = None try: with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: f.write(data) result = parse_json(f.name) finally: if f is not None: os.remove(f.name) assert len(result) == 1 item = result[0] assert item.path == 'pkg/thing.py' assert item.line == 422 assert item.func_name == 'my_function' assert item.type_comments == ['(int) -> None', '(str) -> None'] assert item.samples == 3 class TestTokenize(unittest.TestCase): def test_tokenize(self): # type: () -> None self.assert_tokenize( ' List[int, str] ( )-> *', 'DottedName(List) [ DottedName(int) , DottedName(str) ] ( ) -> * End()') def test_special_cases(self): # type: () -> None self.assert_tokenize('dictionary-itemiterator', 'DottedName(Iterator) End()') self.assert_tokenize('dictionary-keyiterator', 'DottedName(Iterator) End()') self.assert_tokenize('dictionary-valueiterator', 'DottedName(Iterator) End()') self.assert_tokenize('foo-bar', 'DottedName(Any) End()') self.assert_tokenize('pytz.tzfile.Europe/Amsterdam', 'DottedName(datetime.tzinfo) End()') def assert_tokenize(self, s, expected): # type: (str, str) -> None tokens = tokenize(s) actual = ' '.join(str(t) for t in tokens) assert actual == expected def class_arg(name, args=None): # type: (str, Optional[List[AbstractType]]) -> Argument return Argument(ClassType(name, args), ARG_POS) def any_arg(): # type: () -> Argument return Argument(AnyType(), ARG_POS) def tuple_arg(items): # type: (List[AbstractType]) -> Argument return Argument(TupleType(items), ARG_POS) class TestParseTypeComment(unittest.TestCase): def test_empty(self): # type: () -> None self.assert_type_comment('() -> None', ([], ClassType('None'))) def test_simple_args(self): # type: () -> None self.assert_type_comment('(int) -> None', ([class_arg('int')], ClassType('None'))) self.assert_type_comment('(int, str) -> bool', ([class_arg('int'), class_arg('str')], ClassType('bool'))) def test_generic(self): # type: () -> None self.assert_type_comment('(List[int]) -> Dict[str, bool]', ([class_arg('List', [ClassType('int')])], ClassType('Dict', [ClassType('str'), ClassType('bool')]))) def test_any_and_unknown(self): # type: () -> None self.assert_type_comment('(Any) -> pyannotate_runtime.collect_types.UnknownType', ([any_arg()], AnyType())) def test_no_return(self): # type: () -> None self.assert_type_comment('() -> pyannotate_runtime.collect_types.NoReturnType', ([], NoReturnType())) def test_tuple(self): # type: () -> None self.assert_type_comment('(Tuple[]) -> Any', ([tuple_arg([])], AnyType())) self.assert_type_comment('(Tuple[int]) -> Any', ([tuple_arg([ClassType('int')])], AnyType())) self.assert_type_comment('(Tuple[int, str]) -> Any', ([tuple_arg([ClassType('int'), ClassType('str')])], AnyType())) def test_union(self): # type: () -> None self.assert_type_comment('(Union[int, str]) -> Any', ([Argument(UnionType([ClassType('int'), ClassType('str')]), ARG_POS)], AnyType())) self.assert_type_comment('(Union[int]) -> Any', ([class_arg('int')], AnyType())) def test_optional(self): # type: () -> None self.assert_type_comment('(Optional[int]) -> Any', ([Argument(UnionType([ClassType('int'), ClassType('None')]), ARG_POS)], AnyType())) def test_star_args(self): # type: () -> None self.assert_type_comment('(*str) -> Any', ([Argument(ClassType('str'), ARG_STAR)], AnyType())) self.assert_type_comment('(int, *str) -> Any', ([class_arg('int'), Argument(ClassType('str'), ARG_STAR)], AnyType())) def test_star_star_args(self): # type: () -> None self.assert_type_comment('(**str) -> Any', ([Argument(ClassType('str'), ARG_STARSTAR)], AnyType())) self.assert_type_comment('(int, *str, **bool) -> Any', ([class_arg('int'), Argument(ClassType('str'), ARG_STAR), Argument(ClassType('bool'), ARG_STARSTAR)], AnyType())) def test_function(self): # type: () -> None self.assert_type_comment('(function) -> Any', ([class_arg('Callable')], AnyType())) def test_unicode(self): # type: () -> None self.assert_type_comment('(unicode) -> Any', ([class_arg('Text')], AnyType())) def test_bad_annotation(self): # type: () -> None for bad in ['( -> None', '()', ')) -> None', '() -> ', '()->', '() -> None x', 'int', 'int -> None', '(Union[]) -> None', '(List[int) -> None', '(*int, *str) -> None', '(*int, int) -> None', '(**int, *str) -> None', '(**int, str) -> None', '(**int, **str) -> None']: with self.assertRaises(ParseError): parse_type_comment(bad) def assert_type_comment(self, comment, expected): # type: (str, Tuple[List[Argument], AbstractType]) -> None actual = parse_type_comment(comment) assert actual == expected pyannotate-1.2.0/pyannotate_tools/annotations/tests/types_test.py000066400000000000000000000027671353772553400256040ustar00rootroot00000000000000import unittest from pyannotate_tools.annotations.types import AnyType, ClassType, TupleType, UnionType class TestTypes(unittest.TestCase): def test_instance_str(self): # type: () -> None assert str(ClassType('int')) == 'int' assert str(ClassType('List', [ClassType('int')])) == 'List[int]' assert str(ClassType('Dict', [ClassType('int'), ClassType('str')])) == 'Dict[int, str]' def test_any_type_str(self): # type: () -> None assert str(AnyType()) == 'Any' def test_tuple_type_str(self): # type: () -> None assert str(TupleType([ClassType('int')])) == 'Tuple[int]' assert str(TupleType([ClassType('int'), ClassType('str')])) == 'Tuple[int, str]' assert str(TupleType([])) == 'Tuple[()]' def test_union_type_str(Self): # type: () -> None assert str(UnionType([ClassType('int'), ClassType('str')])) == 'Union[int, str]' def test_optional(Self): # type: () -> None assert str(UnionType([ClassType('str'), ClassType('None')])) == 'Optional[str]' assert str(UnionType([ClassType('None'), ClassType('str')])) == 'Optional[str]' assert str(UnionType([ClassType('None'), ClassType('str'), ClassType('int')])) == 'Union[None, str, int]' def test_uniform_tuple_str(self): # type: () -> None assert str(ClassType('Tuple', [ClassType('int')])) == 'Tuple[int, ...]' pyannotate-1.2.0/pyannotate_tools/annotations/types.py000066400000000000000000000071611353772553400233740ustar00rootroot00000000000000"""Internal representation of type objects.""" from typing import NamedTuple, Optional, Sequence class AbstractType(object): """Abstract base class for types.""" class ClassType(AbstractType): """A class type, potentially generic (int, List[str], None, ...)""" def __init__(self, name, args=None): # type: (str, Optional[Sequence[AbstractType]]) -> None self.name = name if args: self.args = tuple(args) else: self.args = () def __repr__(self): # type: () -> str if self.name == 'Tuple' and len(self.args) == 1: return 'Tuple[%s, ...]' % self.args[0] elif self.args: return '%s[%s]' % (self.name, ', '.join(str(arg) for arg in self.args)) else: return self.name def __eq__(self, other): # type: (object) -> bool return isinstance(other, ClassType) and self.name == other.name and self.args == other.args def __hash__(self): # type: () -> int return hash((self.name, self.args)) class AnyType(AbstractType): """The type Any""" def __repr__(self): # type: () -> str return 'Any' def __eq__(self, other): # type: (object) -> bool return isinstance(other, AnyType) def __hash__(self): # type: () -> int return hash('Any') class NoReturnType(AbstractType): """The type mypy_extensions.NoReturn""" def __repr__(self): # type: () -> str return 'mypy_extensions.NoReturn' def __eq__(self, other): # type: (object) -> bool return isinstance(other, NoReturnType) def __hash__(self): # type: () -> int return hash('NoReturn') class TupleType(AbstractType): """Fixed-length tuple Tuple[x, ..., y]""" def __init__(self, items): # type: (Sequence[AbstractType]) -> None self.items = tuple(items) def __repr__(self): # type: () -> str if not self.items: return 'Tuple[()]' # Special case return 'Tuple[%s]' % ', '.join(str(item) for item in self.items) def __eq__(self, other): # type: (object) -> bool return isinstance(other, TupleType) and self.items == other.items def __hash__(self): # type: () -> int return hash(('tuple', self.items)) class UnionType(AbstractType): """Union[x, ..., y]""" def __init__(self, items): # type: (Sequence[AbstractType]) -> None self.items = tuple(items) def __repr__(self): # type: () -> str items = self.items if len(items) == 2: if is_none(items[0]): return 'Optional[%s]' % items[1] elif is_none(items[1]): return 'Optional[%s]' % items[0] return 'Union[%s]' % ', '.join(str(item) for item in items) def __eq__(self, other): # type: (object) -> bool return isinstance(other, UnionType) and set(self.items) == set(other.items) def __hash__(self): # type: () -> int return hash(('union', self.items)) # Argument kind ARG_POS = 'ARG_POS' # Normal ARG_STAR = 'ARG_STAR' # *args ARG_STARSTAR = 'ARG_STARSTAR' # **kwargs # Description of an argument in a signature. The kind is one of ARG_*. Argument = NamedTuple('Argument', [('type', AbstractType), ('kind', str)]) def is_none(t): # type: (AbstractType) -> bool return isinstance(t, ClassType) and t.name == 'None' def is_optional(t): # type: (AbstractType) -> bool return (isinstance(t, UnionType) and len(t.items) == 2 and any(item == ClassType('None') for item in t.items)) pyannotate-1.2.0/pyannotate_tools/fixes/000077500000000000000000000000001353772553400204325ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/fixes/__init__.py000066400000000000000000000000001353772553400225310ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/fixes/fix_annotate.py000066400000000000000000000423601353772553400234700ustar00rootroot00000000000000"""Fixer that inserts mypy annotations into all methods. This transforms e.g. def foo(self, bar, baz=12): return bar + baz into a type annoted version: def foo(self, bar, baz=12): # type: (Any, int) -> Any # noqa: F821 return bar + baz or (when setting options['annotation_style'] to 'py3'): def foo(self, bar : Any, baz : int = 12) -> Any: return bar + baz It does not do type inference but it recognizes some basic default argument values such as numbers and strings (and assumes their type implies the argument type). It also uses some basic heuristics to decide whether to ignore the first argument: - always if it's named 'self' - if there's a @classmethod decorator Finally, it knows that __init__() is supposed to return None. """ from __future__ import print_function import os import re from lib2to3.fixer_base import BaseFix from lib2to3.fixer_util import syms, touch_import, find_indentation from lib2to3.patcomp import compile_pattern from lib2to3.pgen2 import token from lib2to3.pytree import Leaf, Node class FixAnnotate(BaseFix): # This fixer is compatible with the bottom matcher. BM_compatible = True # This fixer shouldn't run by default. explicit = True # The pattern to match. PATTERN = """ funcdef< 'def' name=any parameters=parameters< '(' [args=any] rpar=')' > ':' suite=any+ > """ _maxfixes = os.getenv('MAXFIXES') counter = None if not _maxfixes else int(_maxfixes) def transform(self, node, results): if FixAnnotate.counter is not None: if FixAnnotate.counter <= 0: return # Check if there's already a long-form annotation for some argument. parameters = results.get('parameters') if parameters is not None: for ch in parameters.pre_order(): if ch.prefix.lstrip().startswith('# type:'): return args = results.get('args') if args is not None: for ch in args.pre_order(): if ch.prefix.lstrip().startswith('# type:'): return children = results['suite'][0].children # NOTE: I've reverse-engineered the structure of the parse tree. # It's always a list of nodes, the first of which contains the # entire suite. Its children seem to be: # # [0] NEWLINE # [1] INDENT # [2...n-2] statements (the first may be a docstring) # [n-1] DEDENT # # Comments before the suite are part of the INDENT's prefix. # # "Compact" functions (e.g. "def foo(x, y): return max(x, y)") # have a different structure (no NEWLINE, INDENT, or DEDENT). # Check if there's already an annotation. for ch in children: if ch.prefix.lstrip().startswith('# type:'): return # There's already a # type: comment here; don't change anything. # Python 3 style return annotation are already skipped by the pattern ### Python 3 style argument annotation structure # # Structure of the arguments tokens for one positional argument without default value : # + LPAR '(' # + NAME_NODE_OR_LEAF arg1 # + RPAR ')' # # NAME_NODE_OR_LEAF is either: # 1. Just a leaf with value NAME # 2. A node with children: NAME, ':", node expr or value leaf # # Structure of the arguments tokens for one args with default value or multiple # args, with or without default value, and/or with extra arguments : # + LPAR '(' # + node # [ # + NAME_NODE_OR_LEAF # [ # + EQUAL '=' # + node expr or value leaf # ] # ( # + COMMA ',' # + NAME_NODE_OR_LEAF positional argn # [ # + EQUAL '=' # + node expr or value leaf # ] # )* # ] # [ # + STAR '*' # [ # + NAME_NODE_OR_LEAF positional star argument name # ] # ] # [ # + COMMA ',' # + DOUBLESTAR '**' # + NAME_NODE_OR_LEAF positional keyword argument name # ] # + RPAR ')' # Let's skip Python 3 argument annotations it = iter(args.children) if args else iter([]) for ch in it: if ch.type == token.STAR: # *arg part ch = next(it) if ch.type == token.COMMA: continue elif ch.type == token.DOUBLESTAR: # *arg part ch = next(it) if ch.type > 256: # this is a node, therefore an annotation assert ch.children[0].type == token.NAME return try: ch = next(it) if ch.type == token.COLON: # this is an annotation return elif ch.type == token.EQUAL: ch = next(it) ch = next(it) assert ch.type == token.COMMA continue except StopIteration: break # Compute the annotation annot = self.make_annotation(node, results) if annot is None: return argtypes, restype = annot if self.options['annotation_style'] == 'py3': self.add_py3_annot(argtypes, restype, node, results) else: self.add_py2_annot(argtypes, restype, node, results) # Common to py2 and py3 style annotations: if FixAnnotate.counter is not None: FixAnnotate.counter -= 1 # Also add 'from typing import Any' at the top if needed. self.patch_imports(argtypes + [restype], node) def add_py3_annot(self, argtypes, restype, node, results): args = results.get('args') argleaves = [] if args is None: # function with 0 arguments it = iter([]) elif len(args.children) == 0: # function with 1 argument it = iter([args]) else: # function with multiple arguments or 1 arg with default value it = iter(args.children) for ch in it: argstyle = 'name' if ch.type == token.STAR: # *arg part argstyle = 'star' ch = next(it) if ch.type == token.COMMA: continue elif ch.type == token.DOUBLESTAR: # *arg part argstyle = 'keyword' ch = next(it) assert ch.type == token.NAME argleaves.append((argstyle, ch)) try: ch = next(it) if ch.type == token.EQUAL: ch = next(it) ch = next(it) assert ch.type == token.COMMA continue except StopIteration: break # when self or cls is not annotated, argleaves == argtypes+1 argleaves = argleaves[len(argleaves) - len(argtypes):] for ch_withstyle, chtype in zip(argleaves, argtypes): style, ch = ch_withstyle if style == 'star': assert chtype[0] == '*' assert chtype[1] != '*' chtype = chtype[1:] elif style == 'keyword': assert chtype[0:2] == '**' assert chtype[2] != '*' chtype = chtype[2:] ch.value = '%s: %s' % (ch.value, chtype) # put spaces around the equal sign if ch.next_sibling and ch.next_sibling.type == token.EQUAL: nextch = ch.next_sibling if not nextch.prefix[:1].isspace(): nextch.prefix = ' ' + nextch.prefix nextch = nextch.next_sibling assert nextch != None if not nextch.prefix[:1].isspace(): nextch.prefix = ' ' + nextch.prefix # Add return annotation rpar = results['rpar'] rpar.value = '%s -> %s' % (rpar.value, restype) rpar.changed() def add_py2_annot(self, argtypes, restype, node, results): children = results['suite'][0].children # Insert '# type: {annot}' comment. # For reference, see lib2to3/fixes/fix_tuple_params.py in stdlib. if len(children) >= 1 and children[0].type != token.NEWLINE: # one liner function if children[0].prefix.strip() == '': children[0].prefix = '' children.insert(0, Leaf(token.NEWLINE, '\n')) children.insert( 1, Leaf(token.INDENT, find_indentation(node) + ' ')) children.append(Leaf(token.DEDENT, '')) if len(children) >= 2 and children[1].type == token.INDENT: degen_str = '(...) -> %s' % restype short_str = '(%s) -> %s' % (', '.join(argtypes), restype) if (len(short_str) > 64 or len(argtypes) > 5) and len(short_str) > len(degen_str): self.insert_long_form(node, results, argtypes) annot_str = degen_str else: annot_str = short_str children[1].prefix = '%s# type: %s\n%s' % (children[1].value, annot_str, children[1].prefix) children[1].changed() else: self.log_message("%s:%d: cannot insert annotation for one-line function" % (self.filename, node.get_lineno())) def insert_long_form(self, node, results, argtypes): argtypes = list(argtypes) # We destroy it args = results['args'] if isinstance(args, Node): children = args.children elif isinstance(args, Leaf): children = [args] else: children = [] # Interpret children according to the following grammar: # (('*'|'**')? NAME ['=' expr] ','?)* flag = False # Set when the next leaf should get a type prefix indent = '' # Will be set by the first child def set_prefix(child): if argtypes: arg = argtypes.pop(0).lstrip('*') else: arg = 'Any' # Somehow there aren't enough args if not arg: # Skip self (look for 'check_self' below) prefix = child.prefix.rstrip() else: prefix = ' # type: ' + arg old_prefix = child.prefix.strip() if old_prefix: assert old_prefix.startswith('#') prefix += ' ' + old_prefix child.prefix = prefix + '\n' + indent check_self = self.is_method(node) for child in children: if check_self and isinstance(child, Leaf) and child.type == token.NAME: check_self = False if child.value in ('self', 'cls'): argtypes.insert(0, '') if not indent: indent = ' ' * child.column if isinstance(child, Leaf) and child.value == ',': flag = True elif isinstance(child, Leaf) and flag: set_prefix(child) flag = False need_comma = len(children) >= 1 and children[-1].type != token.COMMA if need_comma and len(children) >= 2: if (children[-1].type == token.NAME and (children[-2].type in (token.STAR, token.DOUBLESTAR))): need_comma = False if need_comma: children.append(Leaf(token.COMMA, u",")) # Find the ')' and insert a prefix before it too. parameters = args.parent close_paren = parameters.children[-1] assert close_paren.type == token.RPAR, close_paren set_prefix(close_paren) assert not argtypes, argtypes def patch_imports(self, types, node): for typ in types: if 'Any' in typ: touch_import('typing', 'Any', node) break def make_annotation(self, node, results): name = results['name'] assert isinstance(name, Leaf), repr(name) assert name.type == token.NAME, repr(name) decorators = self.get_decorators(node) is_method = self.is_method(node) if name.value == '__init__' or not self.has_return_exprs(node): restype = 'None' else: restype = 'Any' args = results.get('args') argtypes = [] if isinstance(args, Node): children = args.children elif isinstance(args, Leaf): children = [args] else: children = [] # Interpret children according to the following grammar: # (('*'|'**')? NAME ['=' expr] ','?)* stars = inferred_type = '' in_default = False at_start = True for child in children: if isinstance(child, Leaf): if child.value in ('*', '**'): stars += child.value elif child.type == token.NAME and not in_default: if not is_method or not at_start or 'staticmethod' in decorators: inferred_type = 'Any' else: # Always skip the first argument if it's named 'self'. # Always skip the first argument of a class method. if child.value == 'self' or 'classmethod' in decorators: pass else: inferred_type = 'Any' elif child.value == '=': in_default = True elif in_default and child.value != ',': if child.type == token.NUMBER: if re.match(r'\d+[lL]?$', child.value): inferred_type = 'int' else: inferred_type = 'float' # TODO: complex? elif child.type == token.STRING: if child.value.startswith(('u', 'U')): inferred_type = 'unicode' else: inferred_type = 'str' elif child.type == token.NAME and child.value in ('True', 'False'): inferred_type = 'bool' elif child.value == ',': if inferred_type: argtypes.append(stars + inferred_type) # Reset stars = inferred_type = '' in_default = False at_start = False if inferred_type: argtypes.append(stars + inferred_type) return argtypes, restype # The parse tree has a different shape when there is a single # decorator vs. when there are multiple decorators. DECORATED = "decorated< (d=decorator | decorators< dd=decorator+ >) funcdef >" decorated = compile_pattern(DECORATED) def get_decorators(self, node): """Return a list of decorators found on a function definition. This is a list of strings; only simple decorators (e.g. @staticmethod) are returned. If the function is undecorated or only non-simple decorators are found, return []. """ if node.parent is None: return [] results = {} if not self.decorated.match(node.parent, results): return [] decorators = results.get('dd') or [results['d']] decs = [] for d in decorators: for child in d.children: if isinstance(child, Leaf) and child.type == token.NAME: decs.append(child.value) return decs def is_method(self, node): """Return whether the node occurs (directly) inside a class.""" node = node.parent while node is not None: if node.type == syms.classdef: return True if node.type == syms.funcdef: return False node = node.parent return False RETURN_EXPR = "return_stmt< 'return' any >" return_expr = compile_pattern(RETURN_EXPR) def has_return_exprs(self, node): """Traverse the tree below node looking for 'return expr'. Return True if at least 'return expr' is found, False if not. (If both 'return' and 'return expr' are found, return True.) """ results = {} if self.return_expr.match(node, results): return True for child in node.children: if child.type not in (syms.funcdef, syms.classdef): if self.has_return_exprs(child): return True return False YIELD_EXPR = "yield_expr< 'yield' [any] >" yield_expr = compile_pattern(YIELD_EXPR) def is_generator(self, node): """Traverse the tree below node looking for 'yield [expr]'.""" results = {} if self.yield_expr.match(node, results): return True for child in node.children: if child.type not in (syms.funcdef, syms.classdef): if self.is_generator(child): return True return False pyannotate-1.2.0/pyannotate_tools/fixes/fix_annotate_json.py000066400000000000000000000257441353772553400245300ustar00rootroot00000000000000"""Fixer that inserts mypy annotations from json file into code. This fixer consumes json from TYPE_COLLECTION_JSON env variable in the following format: [ { "path": "/Users/svorobev/src/client/build_number/__init__.py", "func_name": "is_test", "arg_types": ["int", "str"], "ret_type": "Any" }, ... ] (The old format with "type_comment" instead of "arg_types" and "ret_type" is also still supported.) """ from __future__ import print_function import json # noqa import os import re from lib2to3.fixer_util import syms, touch_import from lib2to3.pgen2 import token from lib2to3.pytree import Base, Leaf, Node from typing import __all__ as typing_all # type: ignore from typing import Any, Dict, List, Optional, Tuple try: from typing import Text except ImportError: # In Python 3.5.1 stdlib, typing.py does not define Text Text = str # type: ignore from .fix_annotate import FixAnnotate # Taken from mypy codebase: # https://github.com/python/mypy/blob/745d300b8304c3dcf601477762bf9d70b9a4619c/mypy/main.py#L503 PY_EXTENSIONS = ['.pyi', '.py'] def crawl_up(arg): # type: (str) -> Tuple[str, str] """Given a .py[i] filename, return (root directory, module). We crawl up the path until we find a directory without __init__.py[i], or until we run out of path components. """ dir, mod = os.path.split(arg) mod = strip_py(mod) or mod while dir and get_init_file(dir): dir, base = os.path.split(dir) if not base: break if mod == '__init__' or not mod: mod = base else: mod = base + '.' + mod return dir, mod def strip_py(arg): # type: (str) -> Optional[str] """Strip a trailing .py or .pyi suffix. Return None if no such suffix is found. """ for ext in PY_EXTENSIONS: if arg.endswith(ext): return arg[:-len(ext)] return None def get_init_file(dir): # type: (str) -> Optional[str] """Check whether a directory contains a file named __init__.py[i]. If so, return the file's name (with dir prefixed). If not, return None. This prefers .pyi over .py (because of the ordering of PY_EXTENSIONS). """ for ext in PY_EXTENSIONS: f = os.path.join(dir, '__init__' + ext) if os.path.isfile(f): return f return None def get_funcname(name, node): # type: (Leaf, Node) -> Text """Get function name by the following rules: - function -> function_name - instance method -> ClassName.function_name """ funcname = name.value if node.parent and node.parent.parent: grand = node.parent.parent if grand.type == syms.classdef: grandname = grand.children[1] assert grandname.type == token.NAME, repr(name) assert isinstance(grandname, Leaf) # Same as previous, for mypy funcname = grandname.value + '.' + funcname return funcname def count_args(node, results): # type: (Node, Dict[str, Base]) -> Tuple[int, bool, bool, bool] """Count arguments and check for self and *args, **kwds. Return (selfish, count, star, starstar) where: - count is total number of args (including *args, **kwds) - selfish is True if the initial arg is named 'self' or 'cls' - star is True iff *args is found - starstar is True iff **kwds is found """ count = 0 selfish = False star = False starstar = False args = results.get('args') if isinstance(args, Node): children = args.children elif isinstance(args, Leaf): children = [args] else: children = [] # Interpret children according to the following grammar: # (('*'|'**')? NAME ['=' expr] ','?)* skip = False previous_token_is_star = False for child in children: if skip: skip = False elif isinstance(child, Leaf): # A single '*' indicates the rest of the arguments are keyword only # and shouldn't be counted as a `*`. if child.type == token.STAR: previous_token_is_star = True elif child.type == token.DOUBLESTAR: starstar = True elif child.type == token.NAME: if count == 0: if child.value in ('self', 'cls'): selfish = True count += 1 if previous_token_is_star: star = True elif child.type == token.EQUAL: skip = True if child.type != token.STAR: previous_token_is_star = False return count, selfish, star, starstar class FixAnnotateJson(FixAnnotate): needed_imports = None def add_import(self, mod, name): if mod == self.current_module(): return if self.needed_imports is None: self.needed_imports = set() self.needed_imports.add((mod, name)) def patch_imports(self, types, node): if self.needed_imports: for mod, name in sorted(self.needed_imports): touch_import(mod, name, node) self.needed_imports = None def current_module(self): return self._current_module def make_annotation(self, node, results): name = results['name'] assert isinstance(name, Leaf), repr(name) assert name.type == token.NAME, repr(name) funcname = get_funcname(name, node) res = self.get_annotation_from_stub(node, results, funcname) return res stub_json_file = os.getenv('TYPE_COLLECTION_JSON') # JSON data for the current file stub_json = None # type: List[Dict[str, Any]] @classmethod def init_stub_json_from_data(cls, data, filename): cls.stub_json = data cls.top_dir, cls._current_module = crawl_up(os.path.abspath(filename)) def init_stub_json(self): with open(self.__class__.stub_json_file) as f: data = json.load(f) self.__class__.init_stub_json_from_data(data, self.filename) def get_annotation_from_stub(self, node, results, funcname): if not self.__class__.stub_json: self.init_stub_json() data = self.__class__.stub_json # We are using relative paths in the JSON. items = [ it for it in data if it['func_name'] == funcname and (it['path'] == self.filename or os.path.join(self.__class__.top_dir, it['path']) == os.path.abspath(self.filename)) ] if len(items) > 1: # this can happen, because of # 1) nested functions # 2) method decorators # as a cheap and dirty solution we just return the nearest one by the line number # (keep the commented-out log_message call in case we need to come back to this) ## self.log_message("%s:%d: duplicate signatures for %s (at lines %s)" % ## (items[0]['path'], node.get_lineno(), items[0]['func_name'], ## ", ".join(str(it['line']) for it in items))) items.sort(key=lambda it: abs(node.get_lineno() - it['line'])) if items: it = items[0] # If the line number is too far off, the source probably drifted # since the trace was collected; it's better to skip this node. # (Allow some drift, since decorators also cause an offset.) if abs(node.get_lineno() - it['line']) >= 5: self.log_message("%s:%d: '%s' signature from line %d too far away -- skipping" % (self.filename, node.get_lineno(), it['func_name'], it['line'])) return None if 'signature' in it: sig = it['signature'] arg_types = sig['arg_types'] # Passes 1-2 don't always understand *args or **kwds, # so add '*Any' or '**Any' at the end if needed. count, selfish, star, starstar = count_args(node, results) for arg_type in arg_types: if arg_type.startswith('**'): starstar = False elif arg_type.startswith('*'): star = False if star: arg_types.append('*Any') if starstar: arg_types.append('**Any') # Pass 1 omits the first arg iff it's named 'self' or 'cls', # even if it's not a method, so insert `Any` as needed # (but only if it's not actually a method). if selfish and len(arg_types) == count - 1: if self.is_method(node): count -= 1 # Leave out the type for 'self' or 'cls' else: arg_types.insert(0, 'Any') # If after those adjustments the count is still off, # print a warning and skip this node. if len(arg_types) != count: self.log_message("%s:%d: source has %d args, annotation has %d -- skipping" % (self.filename, node.get_lineno(), count, len(arg_types))) return None ret_type = sig['return_type'] arg_types = [self.update_type_names(arg_type) for arg_type in arg_types] # Avoid common error "No return value expected" if ret_type == 'None' and self.has_return_exprs(node): ret_type = 'Optional[Any]' # Special case for generators. if (self.is_generator(node) and not (ret_type == 'Iterator' or ret_type.startswith('Iterator['))): if ret_type.startswith('Optional['): assert ret_type[-1] == ']' ret_type = ret_type[9:-1] ret_type = 'Iterator[%s]' % ret_type ret_type = self.update_type_names(ret_type) return arg_types, ret_type return None def update_type_names(self, type_str): # Replace e.g. `List[pkg.mod.SomeClass]` with # `List[SomeClass]` and remember to import it. return re.sub(r'[\w.:]+', self.type_updater, type_str) def type_updater(self, match): # Replace `pkg.mod.SomeClass` with `SomeClass` # and remember to import it. word = match.group() if word == '...': return word if '.' not in word and ':' not in word: # Assume it's either builtin or from `typing` if word in typing_all: self.add_import('typing', word) return word # If there is a :, treat that as the separator between the # module and the class. Otherwise assume everything but the # last element is the module. if ':' in word: mod, name = word.split(':') to_import = name.split('.', 1)[0] else: mod, name = word.rsplit('.', 1) to_import = name self.add_import(mod, to_import) return name pyannotate-1.2.0/pyannotate_tools/fixes/tests/000077500000000000000000000000001353772553400215745ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/fixes/tests/__init__.py000066400000000000000000000000001353772553400236730ustar00rootroot00000000000000pyannotate-1.2.0/pyannotate_tools/fixes/tests/test_annotate_json_py2.py000066400000000000000000000473111353772553400266470ustar00rootroot00000000000000# flake8: noqa # Our flake extension misfires on type comments in strings below. import json import os import tempfile from lib2to3.tests.test_fixers import FixerTestCase from pyannotate_tools.fixes.fix_annotate_json import FixAnnotateJson class TestFixAnnotateJson(FixerTestCase): def setUp(self): super(TestFixAnnotateJson, self).setUp( fix_list=["annotate_json"], fixer_pkg="pyannotate_tools", options={'annotation_style' : 'py2'}, ) # See https://bugs.python.org/issue14243 for details self.tf = tempfile.NamedTemporaryFile(mode='w', delete=False) FixAnnotateJson.stub_json_file = self.tf.name FixAnnotateJson.stub_json = None def tearDown(self): FixAnnotateJson.stub_json = None FixAnnotateJson.stub_json_file = None self.tf.close() os.remove(self.tf.name) super(TestFixAnnotateJson, self).tearDown() def setTestData(self, data): json.dump(data, self.tf) self.tf.close() self.filename = data[0]["path"] def test_basic(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["Foo", "Bar"], "return_type": "Any"}, }]) a = """\ class Foo: pass class Bar: pass def nop(foo, bar): return 42 """ b = """\ from typing import Any class Foo: pass class Bar: pass def nop(foo, bar): # type: (Foo, Bar) -> Any return 42 """ self.check(a, b) def test_keyword_only_argument(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["Foo", "Bar"], "return_type": "Any"}, }]) a = """\ class Foo: pass class Bar: pass def nop(foo, *, bar): return 42 """ b = """\ from typing import Any class Foo: pass class Bar: pass def nop(foo, *, bar): # type: (Foo, Bar) -> Any return 42 """ self.check(a, b) def test_add_typing_import(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, # Check with and without 'typing.' prefix "signature": { "arg_types": ["List[typing.AnyStr]", "Callable[[], int]"], "return_type": "object"}, }]) a = """\ def nop(foo, bar): return 42 """ b = """\ from typing import AnyStr from typing import Callable from typing import List def nop(foo, bar): # type: (List[AnyStr], Callable[[], int]) -> object return 42 """ self.check(a, b) def test_add_other_import(self): self.setTestData( [{"func_name": "nop", "path": "mod1.py", "line": 1, "signature": { "arg_types": ["mod1.MyClass", "mod2.OtherClass"], "return_type": "mod3.AnotherClass"}, }]) a = """\ def nop(foo, bar): return AnotherClass() class MyClass: pass """ b = """\ from mod2 import OtherClass from mod3 import AnotherClass def nop(foo, bar): # type: (MyClass, OtherClass) -> AnotherClass return AnotherClass() class MyClass: pass """ self.check(a, b) def test_add_kwds(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "object"}, }]) a = """\ def nop(foo, **kwds): return 42 """ b = """\ from typing import Any def nop(foo, **kwds): # type: (int, **Any) -> object return 42 """ self.check(a, b) def test_dont_add_kwds(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int", "**AnyStr"], "return_type": "object"}, }]) a = """\ def nop(foo, **kwds): return 42 """ b = """\ from typing import AnyStr def nop(foo, **kwds): # type: (int, **AnyStr) -> object return 42 """ self.check(a, b) def test_add_varargs(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "object"}, }]) a = """\ def nop(foo, *args): return 42 """ b = """\ from typing import Any def nop(foo, *args): # type: (int, *Any) -> object return 42 """ self.check(a, b) def test_dont_add_varargs(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int", "*int"], "return_type": "object"}, }]) a = """\ def nop(foo, *args): return 42 """ b = """\ def nop(foo, *args): # type: (int, *int) -> object return 42 """ self.check(a, b) def test_return_expr_not_none(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "None"}, }]) a = """\ def nop(): return 0 """ b = """\ from typing import Any from typing import Optional def nop(): # type: () -> Optional[Any] return 0 """ self.check(a, b) def test_return_expr_none(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "None"}, }]) a = """\ def nop(): return """ b = """\ def nop(): # type: () -> None return """ self.check(a, b) def test_generator_optional(self): self.setTestData( [{"func_name": "gen", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "Optional[int]"}, }]) a = """\ def gen(): yield 42 """ b = """\ from typing import Iterator def gen(): # type: () -> Iterator[int] yield 42 """ self.check(a, b) def test_generator_plain(self): self.setTestData( [{"func_name": "gen", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def gen(): yield 42 """ b = """\ from typing import Iterator def gen(): # type: () -> Iterator[int] yield 42 """ self.check(a, b) def test_not_generator(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(): def gen(): yield 42 """ b = """\ def nop(): # type: () -> int def gen(): yield 42 """ self.check(a, b) def test_add_self(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(self): pass """ b = """\ from typing import Any def nop(self): # type: (Any) -> int pass """ self.check(a, b) def test_dont_add_self(self): self.setTestData( [{"func_name": "C.nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ class C: def nop(self): pass """ b = """\ class C: def nop(self): # type: () -> int pass """ self.check(a, b) def test_too_many_types(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "int"}, }]) a = """\ def nop(): pass """ self.warns(a, a, "source has 0 args, annotation has 1 -- skipping", unchanged=True) def test_too_few_types(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(a): pass """ self.warns(a, a, "source has 1 args, annotation has 0 -- skipping", unchanged=True) def test_line_number_drift(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 10, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(a): pass """ self.warns(a, a, "signature from line 10 too far away -- skipping", unchanged=True) def test_classmethod(self): # Class method names currently are returned without class name self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int"], "return_type": "int"} }]) a = """\ class C: @classmethod def nop(cls, a): return a """ b = """\ class C: @classmethod def nop(cls, a): # type: (int) -> int return a """ self.check(a, b) def test_staticmethod(self): # Static method names currently are returned without class name self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int"], "return_type": "int"} }]) a = """\ class C: @staticmethod def nop(a): return a """ b = """\ class C: @staticmethod def nop(a): # type: (int) -> int return a """ self.check(a, b) def test_long_form(self): self.maxDiff = None self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]", "*Any"], "return_type": "int"}, }]) a = """\ def nop(a, b, c, # some comment d, e, f, # multi-line # comment g=None, h=0, *args): return 0 """ b = """\ from typing import Any from typing import Optional from typing import Union def nop(a, # type: int b, # type: int c, # type: int # some comment d, # type: str e, # type: str f, # type: str # multi-line # comment g=None, # type: Optional[bool] h=0, # type: Union[int, str] *args # type: Any ): # type: (...) -> int return 0 """ self.check(a, b) def test_long_form_method(self): self.maxDiff = None self.setTestData( [{"func_name": "C.nop", "path": "", "line": 2, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]", "*Any"], "return_type": "int"}, }]) a = """\ class C: def nop(self, a, b, c, # some comment d, e, f, # multi-line # comment g=None, h=0, *args): return 0 """ b = """\ from typing import Any from typing import Optional from typing import Union class C: def nop(self, a, # type: int b, # type: int c, # type: int # some comment d, # type: str e, # type: str f, # type: str # multi-line # comment g=None, # type: Optional[bool] h=0, # type: Union[int, str] *args # type: Any ): # type: (...) -> int return 0 """ self.check(a, b) def test_long_form_classmethod(self): self.maxDiff = None self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]", "*Any"], "return_type": "int"}, }]) a = """\ class C: @classmethod def nop(cls, a, b, c, # some comment d, e, f, g=None, h=0, *args): return 0 """ b = """\ from typing import Any from typing import Optional from typing import Union class C: @classmethod def nop(cls, a, # type: int b, # type: int c, # type: int # some comment d, # type: str e, # type: str f, # type: str g=None, # type: Optional[bool] h=0, # type: Union[int, str] *args # type: Any ): # type: (...) -> int return 0 """ self.check(a, b) # Do the same test for staticmethod a = a.replace('classmethod', 'staticmethod') b = b.replace('classmethod', 'staticmethod') self.check(a, b) def test_long_form_trailing_comma(self): self.maxDiff = None self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]"], "return_type": "int"}, }]) a = """\ def nop(a, b, c, # some comment d, e, f, g=None, h=0): return 0 """ b = """\ from typing import Optional from typing import Union def nop(a, # type: int b, # type: int c, # type: int # some comment d, # type: str e, # type: str f, # type: str g=None, # type: Optional[bool] h=0, # type: Union[int, str] ): # type: (...) -> int return 0 """ self.check(a, b) def test_one_liner(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "int"}, }]) a = """\ def nop(a): return a """ b = """\ def nop(a): # type: (int) -> int return a """ self.check(a, b) def test_variadic(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["Tuple[int, ...]"], "return_type": "int"}, }]) a = """\ def nop(a): return 0 """ b = """\ from typing import Tuple def nop(a): # type: (Tuple[int, ...]) -> int return 0 """ self.check(a, b) def test_nested(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["foo:A.B"], "return_type": "None"}, }]) a = """\ def nop(a): pass """ b = """\ from foo import A def nop(a): # type: (A.B) -> None pass """ self.check(a, b) pyannotate-1.2.0/pyannotate_tools/fixes/tests/test_annotate_json_py3.py000066400000000000000000000432361353772553400266520ustar00rootroot00000000000000# flake8: noqa # Our flake extension misfires on type comments in strings below. import json import os import tempfile from lib2to3.tests.test_fixers import FixerTestCase from pyannotate_tools.fixes.fix_annotate_json import FixAnnotateJson class TestFixAnnotateJson(FixerTestCase): def setUp(self): super(TestFixAnnotateJson, self).setUp( fix_list=["annotate_json"], fixer_pkg="pyannotate_tools", options={'annotation_style' : 'py3'}, ) # See https://bugs.python.org/issue14243 for details self.tf = tempfile.NamedTemporaryFile(mode='w', delete=False) FixAnnotateJson.stub_json_file = self.tf.name FixAnnotateJson.stub_json = None def tearDown(self): FixAnnotateJson.stub_json = None FixAnnotateJson.stub_json_file = None self.tf.close() os.remove(self.tf.name) super(TestFixAnnotateJson, self).tearDown() def setTestData(self, data): json.dump(data, self.tf) self.tf.close() self.filename = data[0]["path"] def test_basic(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["Foo", "Bar"], "return_type": "Any"}, }]) a = """\ class Foo: pass class Bar: pass def nop(foo, bar): return 42 """ b = """\ from typing import Any class Foo: pass class Bar: pass def nop(foo: Foo, bar: Bar) -> Any: return 42 """ self.check(a, b) def test_keyword_only_argument(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["Foo", "Bar"], "return_type": "Any"}, }]) a = """\ class Foo: pass class Bar: pass def nop(foo, *, bar): return 42 """ b = """\ from typing import Any class Foo: pass class Bar: pass def nop(foo: Foo, *, bar: Bar) -> Any: return 42 """ self.check(a, b) def test_add_typing_import(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, # Check with and without 'typing.' prefix "signature": { "arg_types": ["List[typing.AnyStr]", "Callable[[], int]"], "return_type": "object"}, }]) a = """\ def nop(foo, bar): return 42 """ b = """\ from typing import AnyStr from typing import Callable from typing import List def nop(foo: List[AnyStr], bar: Callable[[], int]) -> object: return 42 """ self.check(a, b) def test_add_other_import(self): self.setTestData( [{"func_name": "nop", "path": "mod1.py", "line": 1, "signature": { "arg_types": ["mod1.MyClass", "mod2.OtherClass"], "return_type": "mod3.AnotherClass"}, }]) a = """\ def nop(foo, bar): return AnotherClass() class MyClass: pass """ b = """\ from mod2 import OtherClass from mod3 import AnotherClass def nop(foo: MyClass, bar: OtherClass) -> AnotherClass: return AnotherClass() class MyClass: pass """ self.check(a, b) def test_add_kwds(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "object"}, }]) a = """\ def nop(foo, **kwds): return 42 """ b = """\ from typing import Any def nop(foo: int, **kwds: Any) -> object: return 42 """ self.check(a, b) def test_dont_add_kwds(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int", "**AnyStr"], "return_type": "object"}, }]) a = """\ def nop(foo, **kwds): return 42 """ b = """\ from typing import AnyStr def nop(foo: int, **kwds: AnyStr) -> object: return 42 """ self.check(a, b) def test_add_varargs(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "object"}, }]) a = """\ def nop(foo, *args): return 42 """ b = """\ from typing import Any def nop(foo: int, *args: Any) -> object: return 42 """ self.check(a, b) def test_dont_add_varargs(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int", "*int"], "return_type": "object"}, }]) a = """\ def nop(foo, *args): return 42 """ b = """\ def nop(foo: int, *args: int) -> object: return 42 """ self.check(a, b) def test_return_expr_not_none(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "None"}, }]) a = """\ def nop(): return 0 """ b = """\ from typing import Any from typing import Optional def nop() -> Optional[Any]: return 0 """ self.check(a, b) def test_return_expr_none(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "None"}, }]) a = """\ def nop(): return """ b = """\ def nop() -> None: return """ self.check(a, b) def test_generator_optional(self): self.setTestData( [{"func_name": "gen", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "Optional[int]"}, }]) a = """\ def gen(): yield 42 """ b = """\ from typing import Iterator def gen() -> Iterator[int]: yield 42 """ self.check(a, b) def test_generator_plain(self): self.setTestData( [{"func_name": "gen", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def gen(): yield 42 """ b = """\ from typing import Iterator def gen() -> Iterator[int]: yield 42 """ self.check(a, b) def test_not_generator(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(): def gen(): yield 42 """ b = """\ def nop() -> int: def gen(): yield 42 """ self.check(a, b) def test_add_self(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(self): pass """ b = """\ from typing import Any def nop(self: Any) -> int: pass """ self.check(a, b) def test_dont_add_self(self): self.setTestData( [{"func_name": "C.nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ class C: def nop(self): pass """ b = """\ class C: def nop(self) -> int: pass """ self.check(a, b) def test_too_many_types(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "int"}, }]) a = """\ def nop(): pass """ self.warns(a, a, "source has 0 args, annotation has 1 -- skipping", unchanged=True) def test_too_few_types(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(a): pass """ self.warns(a, a, "source has 1 args, annotation has 0 -- skipping", unchanged=True) def test_line_number_drift(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 10, "signature": { "arg_types": [], "return_type": "int"}, }]) a = """\ def nop(a): pass """ self.warns(a, a, "signature from line 10 too far away -- skipping", unchanged=True) def test_classmethod(self): # Class method names currently are returned without class name self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int"], "return_type": "int"} }]) a = """\ class C: @classmethod def nop(cls, a): return a """ b = """\ class C: @classmethod def nop(cls, a: int) -> int: return a """ self.check(a, b) def test_staticmethod(self): # Static method names currently are returned without class name self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int"], "return_type": "int"} }]) a = """\ class C: @staticmethod def nop(a): return a """ b = """\ class C: @staticmethod def nop(a: int) -> int: return a """ self.check(a, b) def test_long_form(self): self.maxDiff = None self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]", "*Any"], "return_type": "int"}, }]) a = """\ def nop(a, b, c, # some comment d, e, f, # multi-line # comment g=None, h=0, *args): return 0 """ b = """\ from typing import Any from typing import Optional from typing import Union def nop(a: int, b: int, c: int, # some comment d: str, e: str, f: str, # multi-line # comment g: Optional[bool] = None, h: Union[int, str] = 0, *args: Any) -> int: return 0 """ self.check(a, b) def test_long_form_method(self): self.maxDiff = None self.setTestData( [{"func_name": "C.nop", "path": "", "line": 2, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]", "*Any"], "return_type": "int"}, }]) a = """\ class C: def nop(self, a, b, c, # some comment d, e, f, # multi-line # comment g=None, h=0, *args): return 0 """ b = """\ from typing import Any from typing import Optional from typing import Union class C: def nop(self, a: int, b: int, c: int, # some comment d: str, e: str, f: str, # multi-line # comment g: Optional[bool] = None, h: Union[int, str] = 0, *args: Any) -> int: return 0 """ self.check(a, b) def test_long_form_classmethod(self): self.maxDiff = None self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]", "*Any"], "return_type": "int"}, }]) a = """\ class C: @classmethod def nop(cls, a, b, c, # some comment d, e, f, g=None, h=0, *args): return 0 """ b = """\ from typing import Any from typing import Optional from typing import Union class C: @classmethod def nop(cls, a: int, b: int, c: int, # some comment d: str, e: str, f: str, g: Optional[bool] = None, h: Union[int, str] = 0, *args: Any) -> int: return 0 """ self.check(a, b) # Do the same test for staticmethod a = a.replace('classmethod', 'staticmethod') b = b.replace('classmethod', 'staticmethod') self.check(a, b) def test_long_form_trailing_comma(self): self.maxDiff = None self.setTestData( [{"func_name": "nop", "path": "", "line": 3, "signature": { "arg_types": ["int", "int", "int", "str", "str", "str", "Optional[bool]", "Union[int, str]"], "return_type": "int"}, }]) a = """\ def nop(a, b, c, # some comment d, e, f, g=None, h=0): return 0 """ b = """\ from typing import Optional from typing import Union def nop(a: int, b: int, c: int, # some comment d: str, e: str, f: str, g: Optional[bool] = None, h: Union[int, str] = 0) -> int: return 0 """ self.check(a, b) def test_one_liner(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["int"], "return_type": "int"}, }]) a = """\ def nop(a): return a """ b = """\ def nop(a: int) -> int: return a """ self.check(a, b) def test_variadic(self): self.setTestData( [{"func_name": "nop", "path": "", "line": 1, "signature": { "arg_types": ["Tuple[int, ...]"], "return_type": "int"}, }]) a = """\ def nop(a): return 0 """ b = """\ from typing import Tuple def nop(a: Tuple[int, ...]) -> int: return 0 """ self.check(a, b) pyannotate-1.2.0/pyannotate_tools/fixes/tests/test_annotate_py2.py000066400000000000000000000245401353772553400256150ustar00rootroot00000000000000# flake8: noqa # Our flake extension misfires on type comments in strings below. from lib2to3.tests.test_fixers import FixerTestCase # deadcode: fix_annotate is used as part of the fixer_pkg for this test from pyannotate_tools.fixes import fix_annotate class TestFixAnnotate(FixerTestCase): def setUp(self): super(TestFixAnnotate, self).setUp( fix_list=["annotate"], fixer_pkg="pyannotate_tools", options={'annotation_style' : 'py2'}, ) def test_no_arg(self): a = """\ def nop(): return 42 """ b = """\ from typing import Any def nop(): # type: () -> Any return 42 """ self.check(a, b) def test_one_arg(self): a = """\ def incr(arg): return arg+1 """ b = """\ from typing import Any def incr(arg): # type: (Any) -> Any return arg+1 """ self.check(a, b) def test_two_args(self): a = """\ def add(arg1, arg2): return arg1+arg2 """ b = """\ from typing import Any def add(arg1, arg2): # type: (Any, Any) -> Any return arg1+arg2 """ self.check(a, b) def test_defaults(self): a = """\ def foo(iarg=0, farg=0.0, sarg='', uarg=u'', barg=False): return 42 """ b = """\ from typing import Any def foo(iarg=0, farg=0.0, sarg='', uarg=u'', barg=False): # type: (int, float, str, unicode, bool) -> Any return 42 """ self.check(a, b) def test_staticmethod(self): a = """\ class C: @staticmethod def incr(self): return 42 """ b = """\ from typing import Any class C: @staticmethod def incr(self): # type: (Any) -> Any return 42 """ self.check(a, b) def test_classmethod(self): a = """\ class C: @classmethod def incr(cls, arg): return 42 """ b = """\ from typing import Any class C: @classmethod def incr(cls, arg): # type: (Any) -> Any return 42 """ self.check(a, b) def test_instancemethod(self): a = """\ class C: def incr(self, arg): return 42 """ b = """\ from typing import Any class C: def incr(self, arg): # type: (Any) -> Any return 42 """ self.check(a, b) def test_fake_self(self): a = """\ def incr(self, arg): return 42 """ b = """\ from typing import Any def incr(self, arg): # type: (Any, Any) -> Any return 42 """ self.check(a, b) def test_nested_fake_self(self): a = """\ class C: def outer(self): def inner(self, arg): return 42 """ b = """\ from typing import Any class C: def outer(self): # type: () -> None def inner(self, arg): # type: (Any, Any) -> Any return 42 """ self.check(a, b) def test_multiple_decorators(self): a = """\ class C: @contextmanager @classmethod @wrapped('func') def incr(cls, arg): return 42 """ b = """\ from typing import Any class C: @contextmanager @classmethod @wrapped('func') def incr(cls, arg): # type: (Any) -> Any return 42 """ self.check(a, b) def test_stars(self): a = """\ def stuff(*a, **kw): return 4, 2 """ b = """\ from typing import Any def stuff(*a, **kw): # type: (*Any, **Any) -> Any return 4, 2 """ self.check(a, b) def test_idempotency(self): a = """\ def incr(arg): # type: (Any) -> Any return arg+1 """ self.unchanged(a) def test_no_return_expr(self): a = """\ def proc1(arg): return def proc2(arg): pass """ b = """\ from typing import Any def proc1(arg): # type: (Any) -> None return def proc2(arg): # type: (Any) -> None pass """ self.check(a, b) def test_nested_return_expr(self): # The 'return expr' in inner() shouldn't affect the return type of outer(). a = """\ def outer(arg): def inner(): return 42 return """ b = """\ from typing import Any def outer(arg): # type: (Any) -> None def inner(): # type: () -> Any return 42 return """ self.check(a, b) def test_nested_class_return_expr(self): # The 'return expr' in class Inner shouldn't affect the return type of outer(). a = """\ def outer(arg): class Inner: return 42 return """ b = """\ from typing import Any def outer(arg): # type: (Any) -> None class Inner: return 42 return """ self.check(a, b) def test_add_import(self): a = """\ import typing from typing import Callable def incr(arg): return 42 """ b = """\ import typing from typing import Callable from typing import Any def incr(arg): # type: (Any) -> Any return 42 """ self.check(a, b) def test_dont_add_import(self): a = """\ def nop(arg=0): return """ b = """\ def nop(arg=0): # type: (int) -> None return """ self.check(a, b) def test_long_form(self): self.maxDiff = None a = """\ def nop(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8=0, arg9='', *args, **kwds): return """ b = """\ from typing import Any def nop(arg0, # type: Any arg1, # type: Any arg2, # type: Any arg3, # type: Any arg4, # type: Any arg5, # type: Any arg6, # type: Any arg7, # type: Any arg8=0, # type: int arg9='', # type: str *args, # type: Any **kwds # type: Any ): # type: (...) -> None return """ self.check(a, b) def test_long_form_trailing_comma(self): self.maxDiff = None a = """\ def nop(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7=None, arg8=0, arg9='', arg10=False,): return """ b = """\ from typing import Any def nop(arg0, # type: Any arg1, # type: Any arg2, # type: Any arg3, # type: Any arg4, # type: Any arg5, # type: Any arg6, # type: Any arg7=None, # type: Any arg8=0, # type: int arg9='', # type: str arg10=False, # type: bool ): # type: (...) -> None return """ self.check(a, b) def test_one_liner(self): a = """\ class C: def nop(self, a): a = a; return a # Something # More pass """ b = """\ from typing import Any class C: def nop(self, a): # type: (Any) -> Any a = a; return a # Something # More pass """ self.check(a, b) def test_idempotency_long_1arg(self): a = """\ def nop(a # type: int ): pass """ self.unchanged(a) def test_idempotency_long_1arg_comma(self): a = """\ def nop(a, # type: int ): pass """ self.unchanged(a) def test_idempotency_long_2args_first(self): a = """\ def nop(a, # type: int b): pass """ self.unchanged(a) def test_idempotency_long_2args_last(self): a = """\ def nop(a, b # type: int ): pass """ self.unchanged(a) def test_idempotency_long_varargs(self): a = """\ def nop(*a # type: int ): pass """ self.unchanged(a) def test_idempotency_long_kwargs(self): a = """\ def nop(**a # type: int ): pass """ self.unchanged(a) pyannotate-1.2.0/pyannotate_tools/fixes/tests/test_annotate_py3.py000066400000000000000000000367041353772553400256230ustar00rootroot00000000000000# flake8: noqa # Our flake extension misfires on type comments in strings below. from lib2to3.tests.test_fixers import FixerTestCase import unittest # deadcode: fix_annotate is used as part of the fixer_pkg for this test from pyannotate_tools.fixes import fix_annotate class TestFixAnnotate3(FixerTestCase): def setUp(self): super(TestFixAnnotate3, self).setUp( fix_list=["annotate"], fixer_pkg="pyannotate_tools", options={'annotation_style' : 'py3'} ) def test_no_arg_1(self) : a = """\ def nop(): return 42 """ b = """\ from typing import Any def nop() -> Any: return 42 """ self.check(a, b) def test_no_arg_2(self) : a = """\ def nop(): return 42 """ b = """\ from typing import Any def nop() -> Any: return 42 """ self.check(a, b) def test_no_arg_3(self) : a = """\ def nop( ): return 42 """ b = """\ from typing import Any def nop( ) -> Any: return 42 """ self.check(a, b) def test_no_arg_4(self) : a = """\ def nop( ) \ : return 42 """ b = """\ from typing import Any def nop( ) -> Any \ : return 42 """ self.check(a, b) def test_no_arg_5(self) : a = """\ def nop( # blah ): # blah return 42 # blah """ b = """\ from typing import Any def nop( # blah ) -> Any: # blah return 42 # blah """ self.check(a, b) def test_no_arg_6(self) : a = """\ def nop( # blah ) \ : # blah return 42 # blah """ b = """\ from typing import Any def nop( # blah ) -> Any \ : # blah return 42 # blah """ self.check(a, b) def test_one_arg_1(self): a = """\ def incr(arg): return arg+1 """ b = """\ from typing import Any def incr(arg: Any) -> Any: return arg+1 """ self.check(a, b) def test_one_arg_2(self): a = """\ def incr(arg=0): return arg+1 """ b = """\ from typing import Any def incr(arg: int = 0) -> Any: return arg+1 """ self.check(a, b) def test_one_arg_3(self): a = """\ def incr( arg=0 ): return arg+1 """ b = """\ from typing import Any def incr( arg: int = 0 ) -> Any: return arg+1 """ self.check(a, b) def test_one_arg_4(self): a = """\ def incr( arg = 0 ): return arg+1 """ b = """\ from typing import Any def incr( arg: int = 0 ) -> Any: return arg+1 """ self.check(a, b) def test_two_args_1(self): a = """\ def add(arg1, arg2): return arg1+arg2 """ b = """\ from typing import Any def add(arg1: Any, arg2: Any) -> Any: return arg1+arg2 """ self.check(a, b) def test_two_args_2(self): a = """\ def add(arg1=0, arg2=0.1): return arg1+arg2 """ b = """\ from typing import Any def add(arg1: int = 0, arg2: float = 0.1) -> Any: return arg1+arg2 """ self.check(a, b) def test_two_args_3(self): a = """\ def add(arg1, arg2=0.1): return arg1+arg2 """ b = """\ from typing import Any def add(arg1: Any, arg2: float = 0.1) -> Any: return arg1+arg2 """ def test_two_args_4(self): a = """\ def add(arg1, arg2 = 0.1): return arg1+arg2 """ b = """\ from typing import Any def add(arg1: Any, arg2: float = 0.1) -> Any: return arg1+arg2 """ self.check(a, b) self.check(a, b) def test_defaults_1(self): a = """\ def foo(iarg=0, farg=0.0, sarg='', uarg=u'', barg=False): return 42 """ b = """\ from typing import Any def foo(iarg: int = 0, farg: float = 0.0, sarg: str = '', uarg: unicode = u'', barg: bool = False) -> Any: return 42 """ self.check(a, b) def test_defaults_2(self): a = """\ def foo(iarg=0, farg=0.0, sarg='', uarg=u'', barg=False, targ=(1,2,3)): return 42 """ b = """\ from typing import Any def foo(iarg: int = 0, farg: float = 0.0, sarg: str = '', uarg: unicode = u'', barg: bool = False, targ: Any = (1,2,3)) -> Any: return 42 """ self.check(a, b) def test_defaults_3(self): a = """\ def foo(iarg=0, farg, sarg='', uarg, barg=False, targ=(1,2,3)): return 42 """ b = """\ from typing import Any def foo(iarg: int = 0, farg: Any, sarg: str = '', uarg: Any, barg: bool = False, targ: Any = (1,2,3)) -> Any: return 42 """ self.check(a, b) def test_staticmethod(self): a = """\ class C: @staticmethod def incr(self): return 42 """ b = """\ from typing import Any class C: @staticmethod def incr(self: Any) -> Any: return 42 """ self.check(a, b) def test_classmethod(self): a = """\ class C: @classmethod def incr(cls, arg): return 42 """ b = """\ from typing import Any class C: @classmethod def incr(cls, arg: Any) -> Any: return 42 """ self.check(a, b) def test_instancemethod(self): a = """\ class C: def incr(self, arg): return 42 """ b = """\ from typing import Any class C: def incr(self, arg: Any) -> Any: return 42 """ self.check(a, b) def test_fake_self(self): a = """\ def incr(self, arg): return 42 """ b = """\ from typing import Any def incr(self: Any, arg: Any) -> Any: return 42 """ self.check(a, b) def test_nested_fake_self(self): a = """\ class C: def outer(self): def inner(self, arg): return 42 """ b = """\ from typing import Any class C: def outer(self) -> None: def inner(self: Any, arg: Any) -> Any: return 42 """ self.check(a, b) def test_multiple_decorators(self): a = """\ class C: @contextmanager @classmethod @wrapped('func') def incr(cls, arg): return 42 """ b = """\ from typing import Any class C: @contextmanager @classmethod @wrapped('func') def incr(cls, arg: Any) -> Any: return 42 """ self.check(a, b) def test_stars_1(self): a = """\ def stuff(*a): return 4, 2 """ b = """\ from typing import Any def stuff(*a: Any) -> Any: return 4, 2 """ self.check(a, b) def test_stars_2(self): a = """\ def stuff(a, *b): return 4, 2 """ b = """\ from typing import Any def stuff(a: Any, *b: Any) -> Any: return 4, 2 """ self.check(a, b) def test_keywords_1(self): a = """\ def stuff(**kw): return 4, 2 """ b = """\ from typing import Any def stuff(**kw: Any) -> Any: return 4, 2 """ self.check(a, b) def test_keywords_2(self): a = """\ def stuff(a, **kw): return 4, 2 """ b = """\ from typing import Any def stuff(a: Any, **kw: Any) -> Any: return 4, 2 """ self.check(a, b) def test_keywords_3(self): a = """\ def stuff(a, *b, **kw): return 4, 2 """ b = """\ from typing import Any def stuff(a: Any, *b: Any, **kw: Any) -> Any: return 4, 2 """ self.check(a, b) def test_keywords_4(self): a = """\ def stuff(*b, **kw): return 4, 2 """ b = """\ from typing import Any def stuff(*b: Any, **kw: Any) -> Any: return 4, 2 """ self.check(a, b) def test_no_return_expr(self): a = """\ def proc1(arg): return def proc2(arg): pass """ b = """\ from typing import Any def proc1(arg: Any) -> None: return def proc2(arg: Any) -> None: pass """ self.check(a, b) def test_nested_return_expr(self): # The 'return expr' in inner() shouldn't affect the return type of outer(). a = """\ def outer(arg): def inner(): return 42 return """ b = """\ from typing import Any def outer(arg: Any) -> None: def inner() -> Any: return 42 return """ self.check(a, b) def test_nested_class_return_expr(self): # The 'return expr' in class Inner shouldn't affect the return type of outer(). a = """\ def outer(arg): class Inner: return 42 return """ b = """\ from typing import Any def outer(arg: Any) -> None: class Inner: return 42 return """ self.check(a, b) def test_add_import(self): a = """\ import typing from typing import Callable def incr(arg): return 42 """ b = """\ import typing from typing import Callable from typing import Any def incr(arg: Any) -> Any: return 42 """ self.check(a, b) def test_dont_add_import(self): a = """\ def nop(arg=0): return """ b = """\ def nop(arg: int = 0) -> None: return """ self.check(a, b) def test_long_form(self): self.maxDiff = None a = """\ def nop(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8=0, arg9='', *args, **kwds): return """ b = """\ from typing import Any def nop(arg0: Any, arg1: Any, arg2: Any, arg3: Any, arg4: Any, arg5: Any, arg6: Any, arg7: Any, arg8: int = 0, arg9: str = '', *args: Any, **kwds: Any) -> None: return """ self.check(a, b) def test_long_form_trailing_comma(self): self.maxDiff = None a = """\ def nop(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7=None, arg8=0, arg9='', arg10=False,): return """ b = """\ from typing import Any def nop(arg0: Any, arg1: Any, arg2: Any, arg3: Any, arg4: Any, arg5: Any, arg6: Any, arg7: Any = None, arg8: int = 0, arg9: str = '', arg10: bool = False,) -> None: return """ self.check(a, b) def test_one_liner(self): a = """\ class C: def nop(self, a): a = a; return a # Something # More pass """ b = """\ from typing import Any class C: def nop(self, a: Any) -> Any: a = a; return a # Something # More pass """ self.check(a, b) def test_idempotency_long_1arg(self): a = """\ def nop(a: int ): pass """ self.unchanged(a) def test_idempotency_long_1arg_comma(self): a = """\ def nop(a: int, ): pass """ self.unchanged(a) def test_idempotency_long_2args_first(self): a = """\ def nop(a: int, b): pass """ self.unchanged(a) def test_idempotency_long_2args_last(self): a = """\ def nop(a, b: int ): pass """ self.unchanged(a) def test_idempotency_long_varargs(self): a = """\ def nop(*a: int ): pass """ self.unchanged(a) def test_idempotency_long_kwargs(self): a = """\ def nop(**a: int ): pass """ self.unchanged(a) def test_idempotency_arg0_ret_value(self): a = """\ def nop() -> int: pass """ self.unchanged(a) def test_idempotency_arg1_ret_value(self): a = """\ def nop(a) -> int: pass """ self.unchanged(a) def test_idempotency_arg1_default_1(self): a = """\ def nop(a: int=0): pass """ self.unchanged(a) def test_idempotency_arg1_default_2(self): a = """\ def nop(a: List[int]=[]): pass """ self.unchanged(a) def test_idempotency_arg1_default_3(self): a = """\ def nop(a: List[int]=[1,2,3]): pass """ self.unchanged(a) pyannotate-1.2.0/requirements.txt000066400000000000000000000001521353772553400171740ustar00rootroot00000000000000mypy_extensions>=0.3.0 pytest>=3.3.0 setuptools>=28.8.0 six>=1.11.0 typing>=3.6.2; python_version < '3.5' pyannotate-1.2.0/setup.cfg000066400000000000000000000000341353772553400155300ustar00rootroot00000000000000[bdist_wheel] universal = 1 pyannotate-1.2.0/setup.py000066400000000000000000000032101353772553400154200ustar00rootroot00000000000000#!/usr/bin/env python import os from setuptools import setup def get_long_description(): filename = os.path.join(os.path.dirname(__file__), 'README.md') with open(filename) as f: return f.read() setup(name='pyannotate', version='1.2.0', description="PyAnnotate: Auto-generate PEP-484 annotations", long_description=get_long_description(), long_description_content_type="text/markdown", author='Dropbox', author_email='guido@dropbox.com', url='https://github.com/dropbox/pyannotate', license='Apache 2.0', platforms=['POSIX'], packages=['pyannotate_runtime', 'pyannotate_tools', 'pyannotate_tools.annotations', 'pyannotate_tools.fixes'], entry_points={'console_scripts': ['pyannotate=pyannotate_tools.annotations.__main__:main']}, classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Software Development', ], install_requires = ['six', 'mypy_extensions', 'typing >= 3.5.3; python_version < "3.5"' ], ) pyannotate-1.2.0/tests/000077500000000000000000000000001353772553400150545ustar00rootroot00000000000000pyannotate-1.2.0/tests/integration_test.py000066400000000000000000000112101353772553400210030ustar00rootroot00000000000000"""Some things you just can't test as unit tests""" import os import subprocess import sys import tempfile import unittest import shutil example = """ def main(): print(gcd(15, 10)) print(gcd(45, 12)) def gcd(a, b): while b: a, b = b, a%b return a """ driver = """ from pyannotate_runtime import collect_types if __name__ == '__main__': collect_types.init_types_collection() with collect_types.collect(): main() collect_types.dump_stats('type_info.json') """ class_example = """ class A(object): pass def f(x): return x def main(): f(A()) f(A()) """ class IntegrationTest(unittest.TestCase): def setUp(self): self.savedir = os.getcwd() os.putenv('PYTHONPATH', self.savedir) self.tempdir = tempfile.mkdtemp() os.chdir(self.tempdir) def tearDown(self): os.chdir(self.savedir) shutil.rmtree(self.tempdir) def test_simple(self): with open('gcd.py', 'w') as f: f.write(example) with open('driver.py', 'w') as f: f.write('from gcd import main\n') f.write(driver) subprocess.check_call([sys.executable, 'driver.py']) output = subprocess.check_output([sys.executable, '-m', 'pyannotate_tools.annotations', 'gcd.py']) lines = output.splitlines() assert b'+ # type: () -> None' in lines assert b'+ # type: (int, int) -> int' in lines def test_auto_any(self): with open('gcd.py', 'w') as f: f.write(example) output = subprocess.check_output([sys.executable, '-m', 'pyannotate_tools.annotations', '-a', 'gcd.py']) lines = output.splitlines() assert b'+ # type: () -> None' in lines assert b'+ # type: (Any, Any) -> Any' in lines def test_no_type_info(self): with open('gcd.py', 'w') as f: f.write(example) try: subprocess.check_output([sys.executable, '-m', 'pyannotate_tools.annotations', 'gcd.py'], stderr=subprocess.STDOUT) assert False, "Expected an error" except subprocess.CalledProcessError as err: assert err.returncode == 1 lines = err.output.splitlines() assert (b"Can't open type info file: " b"[Errno 2] No such file or directory: 'type_info.json'" in lines) def test_package(self): os.makedirs('foo') with open('foo/__init__.py', 'w') as f: pass with open('foo/gcd.py', 'w') as f: f.write(example) with open('driver.py', 'w') as f: f.write('from foo.gcd import main\n') f.write(driver) subprocess.check_call([sys.executable, 'driver.py']) output = subprocess.check_output([sys.executable, '-m', 'pyannotate_tools.annotations', 'foo/gcd.py']) lines = output.splitlines() assert b'+ # type: () -> None' in lines assert b'+ # type: (int, int) -> int' in lines def test_subdir(self): os.makedirs('foo') with open('foo/gcd.py', 'w') as f: f.write(example) with open('driver.py', 'w') as f: f.write('import sys\n') f.write('sys.path.insert(0, "foo")\n') f.write('from gcd import main\n') f.write(driver) subprocess.check_call([sys.executable, 'driver.py']) output = subprocess.check_output([sys.executable, '-m', 'pyannotate_tools.annotations', # Construct platform-correct pathname: os.path.join('foo', 'gcd.py')]) lines = output.splitlines() assert b'+ # type: () -> None' in lines assert b'+ # type: (int, int) -> int' in lines def test_subdir_w_class(self): os.makedirs('foo') with open('foo/bar.py', 'w') as f: f.write(class_example) with open('driver.py', 'w') as f: f.write('import sys\n') f.write('sys.path.insert(0, "foo")\n') f.write('from bar import main\n') f.write(driver) subprocess.check_call([sys.executable, 'driver.py']) output = subprocess.check_output([sys.executable, '-m', 'pyannotate_tools.annotations', # Construct platform-correct pathname: os.path.join('foo', 'bar.py')]) lines = output.splitlines() print(b'\n'.join(lines).decode()) assert b'+ # type: () -> None' in lines assert b'+ # type: (A) -> A' in lines assert not any(line.startswith(b'+') and b'import' in line for line in lines)