pax_global_header 0000666 0000000 0000000 00000000064 14101703655 0014514 g ustar 00root root 0000000 0000000 52 comment=4e4d587e0e023eafb9fb935c210aad35ac04e0d3
beniget-0.4.1/ 0000775 0000000 0000000 00000000000 14101703655 0013133 5 ustar 00root root 0000000 0000000 beniget-0.4.1/.github/ 0000775 0000000 0000000 00000000000 14101703655 0014473 5 ustar 00root root 0000000 0000000 beniget-0.4.1/.github/workflows/ 0000775 0000000 0000000 00000000000 14101703655 0016530 5 ustar 00root root 0000000 0000000 beniget-0.4.1/.github/workflows/core.yml 0000664 0000000 0000000 00000001260 14101703655 0020202 0 ustar 00root root 0000000 0000000 name: core
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-18.04
strategy:
matrix:
python-version: [2.7, 3.6, 3.8, 3.9, 3.10-dev]
steps:
- uses: actions/checkout@v2
- name: Setup Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Setup
run: |
python setup.py install
- name: Testing sequential
run: |
python setup.py test
beniget-0.4.1/.gitignore 0000664 0000000 0000000 00000000065 14101703655 0015124 0 ustar 00root root 0000000 0000000 *.pyc
.eggs
beniget.egg-info
.pytest_cache
.vscode
beniget-0.4.1/LICENSE 0000664 0000000 0000000 00000002722 14101703655 0014143 0 ustar 00root root 0000000 0000000 Copyright (c) 2019, Serge Guelton
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
Neither the name of HPCProject, Serge Guelton nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
beniget-0.4.1/MANIFEST.in 0000664 0000000 0000000 00000000106 14101703655 0014666 0 ustar 00root root 0000000 0000000 include LICENSE
include requirements.txt
recursive-include tests *.py
beniget-0.4.1/README.rst 0000664 0000000 0000000 00000020746 14101703655 0014633 0 ustar 00root root 0000000 0000000 Gast, Beniget!
==============
Beniget is a collection of Compile-time analyse on Python Abstract Syntax Tree(AST).
It's a building block to write static analyzer or compiler for Python.
Beniget relies on `gast `_ to provide a cross
version abstraction of the AST, effectively working on both Python2 and
Python3.
API
---
Basically Beniget provides three analyse:
- ``beniget.Ancestors`` that maps each node to the list of enclosing nodes;
- ``beniget.DefUseChains`` that maps each node to the list of definition points in that node;
- ``beniget.UseDefChains`` that maps each node to the list of possible definition of that node.
See sample usages and/or run ``pydoc beniget`` for more information :-).
Sample Usages
-------------
Detect unused imports
*********************
This is a very basic usage: look for def without any use, and warn about them, focusing on imported values.
.. code:: python
>>> import beniget, gast as ast
# parse some simple statements
>>> code = "from math import cos, sin; print(cos(3))"
>>> module = ast.parse(code)
# compute the def-use chains at module level
>>> duc = beniget.DefUseChains()
>>> duc.visit(module)
# grab the import statement
>>> imported = module.body[0].names
# inspect the users of each imported name
>>> for name in imported:
... ud = duc.chains[name]
... if not ud.users():
... print("Unused import: {}".format(ud.name()))
Unused import: sin
*NOTE*: Due to the dynamic nature of Python, one can fool this analysis by
calling the ``eval`` function, eventually through an indirection, or by performing a lookup
into ``globals()``.
Find all functions marked with a given decorator
************************************************
Let's assume we've got a ``@nice`` decorator applied to some functions. We can traverse the users
of this decorator to find which functions are decorated.
.. code:: python
# parse some simple statements
>>> code = """
... nice = lambda x: x
... @nice
... def aw(): pass
... def some(): pass"""
>>> module = ast.parse(code)
# compute the def-use chains at module level
>>> duc = beniget.DefUseChains()
>>> duc.visit(module)
# analysis to find parent of a node
>>> ancestors = beniget.Ancestors()
>>> ancestors.visit(module)
# find the nice definition
>>> nice = [d for d in duc.locals[module] if d.name() == "nice"][0]
# walkthrough its users
>>> for use in nice.users():
... # we're interested in the parent of the decorator
... parents = ancestors.parents(use.node)
... # direct parent of the decorator is the function
... fdef = parents[-1]
... print(fdef.name)
aw
Gather attributes of ``self``
*****************************
This analysis gathers all attributes of a class, by going through all methods and checking
the users of the first method parameter, investigating the one used in attribute lookup.
.. code:: python
>>> import gast as ast
>>> import beniget
>>> class Attributes(ast.NodeVisitor):
...
... def __init__(self, module_node):
... # compute the def-use of the module
... self.chains = beniget.DefUseChains()
... self.chains.visit(module_node)
... self.users = set() # all users of `self`
... self.attributes = set() # attributes of current class
...
... def visit_ClassDef(self, node):
... # walk methods and fill users of `self`
... for stmt in node.body:
... if isinstance(stmt, ast.FunctionDef):
... self_def = self.chains.chains[stmt.args.args[0]]
... self.users.update(use.node for use in self_def.users())
... self.generic_visit(node)
...
... def visit_Attribute(self, node):
... # any attribute of `self` is registered
... if node.value in self.users:
... self.attributes.add(node.attr)
>>> code = "class My(object):\n def __init__(self, x): self.x = x"
>>> module = ast.parse(code)
>>> classdef = module.body[0]
>>> attr = Attributes(module)
>>> attr.visit(classdef)
>>> list(attr.attributes)
['x']
*NOTE*: This is *not* an alias analysis, so assigning ``self`` to another variable, or
setting it in a tuple is not captured by this analysis. It's still possible to write such an
a analysis using def-use chains though ;-)
Compute the identifiers captured by a function
**********************************************
In Python, inner functions (and lambdas) can capture identifiers defined in the outer scope.
This analysis computes such identifiers by registering each identifier defined in the function,
then walking through all loaded identifier and checking whether it's local or not.
.. code:: python
>>> import gast as ast
>>> import beniget
>>> class Capture(ast.NodeVisitor):
...
... def __init__(self, module_node):
... # initialize def-use chains
... self.chains = beniget.DefUseChains()
... self.chains.visit(module_node)
... self.users = set() # users of local definitions
... self.captured = set() # identifiers that don't belong to local users
...
... def visit_FunctionDef(self, node):
... # initialize the set of node using a local variable
... for def_ in self.chains.locals[node]:
... self.users.update(use.node for use in def_.users())
... self.generic_visit(node)
...
... def visit_Name(self, node):
... # register load of identifiers not locally definied
... if isinstance(node.ctx, ast.Load):
... if node not in self.users:
... self.captured.add(node.id)
>>> code = 'def foo(x):\n def bar(): return x\n return bar'
>>> module = ast.parse(code)
>>> inner_function = module.body[0].body[0]
>>> capture = Capture(module)
>>> capture.visit(inner_function)
>>> list(capture.captured)
['x']
Compute the set of instructions required to compute a function
**************************************************************
This is actually very similar to the computation of the closure, but this time
let's use the UseDef chains combined with the ancestors.
.. code:: python
>>> import gast as ast
>>> import beniget
>>> class CaptureX(ast.NodeVisitor):
...
... def __init__(self, module_node, fun):
... self.fun = fun
... # initialize use-def chains
... du = beniget.DefUseChains()
... du.visit(module_node)
... self.chains = beniget.UseDefChains(du)
... self.ancestors = beniget.Ancestors()
... self.ancestors.visit(module_node)
... self.external = list()
... self.visited_external = set()
...
... def visit_Name(self, node):
... # register load of identifiers not locally defined
... if isinstance(node.ctx, ast.Load):
... uses = self.chains.chains[node]
... for use in uses:
... try:
... parents = self.ancestors.parents(use.node)
... except KeyError:
... return # a builtin
... if self.fun not in parents:
... parent = self.ancestors.parentStmt(use.node)
... if parent not in self.visited_external:
... self.visited_external.add(parent)
... self.external.append(parent)
... self.rec(parent)
...
... def rec(self, node):
... "walk definitions to find their operands's def"
... if isinstance(node, ast.Assign):
... self.visit(node.value)
... # TODO: implement this for AugAssign etc
>>> code = 'a = 1; b = [a, a]; c = len(b)\ndef foo():\n return c'
>>> module = ast.parse(code)
>>> function = module.body[3]
>>> capturex = CaptureX(module, function)
>>> capturex.visit(function)
>>> # the three top level assignments have been captured!
>>> list(map(type, capturex.external))
[, , ]
Acknowledgments
---------------
Beniget is in Pierre Augier's debt, for he triggered the birth of beniget and provided
countless meaningful bug reports and advices. Trugarez!
beniget-0.4.1/beniget/ 0000775 0000000 0000000 00000000000 14101703655 0014550 5 ustar 00root root 0000000 0000000 beniget-0.4.1/beniget/__init__.py 0000664 0000000 0000000 00000000151 14101703655 0016656 0 ustar 00root root 0000000 0000000 from __future__ import absolute_import
from beniget.beniget import Ancestors, DefUseChains, UseDefChains
beniget-0.4.1/beniget/beniget.py 0000664 0000000 0000000 00000103627 14101703655 0016550 0 ustar 00root root 0000000 0000000 from collections import defaultdict, OrderedDict
from contextlib import contextmanager
import sys
import gast as ast
class ordered_set(object):
def __init__(self, elements=None):
self.values = OrderedDict.fromkeys(elements or [])
def add(self, value):
self.values[value] = None
def update(self, values):
self.values.update((k, None) for k in values)
def __iter__(self):
return iter(self.values.keys())
def __contains__(self, value):
return value in self.values
def __add__(self, other):
out = self.values.copy()
out.update(other.values)
return out
def __len__(self):
return len(self.values)
class Ancestors(ast.NodeVisitor):
"""
Build the ancestor tree, that associates a node to the list of node visited
from the root node (the Module) to the current node
>>> import gast as ast
>>> code = 'def foo(x): return x + 1'
>>> module = ast.parse(code)
>>> from beniget import Ancestors
>>> ancestors = Ancestors()
>>> ancestors.visit(module)
>>> binop = module.body[0].body[0].value
>>> for n in ancestors.parents(binop):
... print(type(n))
"""
def __init__(self):
self._parents = dict()
self._current = list()
def generic_visit(self, node):
self._parents[node] = list(self._current)
self._current.append(node)
super(Ancestors, self).generic_visit(node)
self._current.pop()
def parent(self, node):
return self._parents[node][-1]
def parents(self, node):
return self._parents[node]
def parentInstance(self, node, cls):
for n in reversed(self._parents[node]):
if isinstance(n, cls):
return n
raise ValueError("{} has no parent of type {}".format(node, cls))
def parentFunction(self, node):
return self.parentInstance(node, (ast.FunctionDef,
ast.AsyncFunctionDef))
def parentStmt(self, node):
return self.parentInstance(node, ast.stmt)
class Def(object):
"""
Model a definition, either named or unnamed, and its users.
"""
__slots__ = "node", "_users"
def __init__(self, node):
self.node = node
self._users = ordered_set()
def add_user(self, node):
assert isinstance(node, Def)
self._users.add(node)
def name(self):
"""
If the node associated to this Def has a name, returns this name.
Otherwise returns its type
"""
if isinstance(self.node, (ast.ClassDef,
ast.FunctionDef,
ast.AsyncFunctionDef)):
return self.node.name
elif isinstance(self.node, ast.Name):
return self.node.id
elif isinstance(self.node, ast.alias):
base = self.node.name.split(".", 1)[0]
return self.node.asname or base
elif isinstance(self.node, tuple):
return self.node[1]
else:
return type(self.node).__name__
def users(self):
"""
The list of ast entity that holds a reference to this node
"""
return self._users
def __repr__(self):
return self._repr({})
def _repr(self, nodes):
if self in nodes:
return "(#{})".format(nodes[self])
else:
nodes[self] = len(nodes)
return "{} -> ({})".format(
self.node, ", ".join(u._repr(nodes.copy())
for u in self._users)
)
def __str__(self):
return self._str({})
def _str(self, nodes):
if self in nodes:
return "(#{})".format(nodes[self])
else:
nodes[self] = len(nodes)
return "{} -> ({})".format(
self.name(), ", ".join(u._str(nodes.copy())
for u in self._users)
)
Builtins = {}
if sys.version_info.major == 2:
BuiltinsSrc = __builtins__
else:
import builtins
BuiltinsSrc = builtins.__dict__
Builtins = {k: v for k, v in BuiltinsSrc.items()}
Builtins["__file__"] = __file__
DeclarationStep, DefinitionStep = object(), object()
class CollectGlobals(ast.NodeVisitor):
def __init__(self):
self.Globals = defaultdict(list)
def visit_Global(self, node):
for name in node.names:
self.Globals[name].append((node, name))
class DefUseChains(ast.NodeVisitor):
"""
Module visitor that gathers two kinds of informations:
- locals: Dict[node, List[Def]], a mapping between a node and the list
of variable defined in this node,
- chains: Dict[node, Def], a mapping between nodes and their chains.
>>> import gast as ast
>>> module = ast.parse("from b import c, d; c()")
>>> duc = DefUseChains()
>>> duc.visit(module)
>>> for head in duc.locals[module]:
... print("{}: {}".format(head.name(), len(head.users())))
c: 1
d: 0
>>> alias_def = duc.chains[module.body[0].names[0]]
>>> print(alias_def)
c -> (c -> (Call -> ()))
"""
def __init__(self, filename=None):
"""
- filename: str, included in error messages if specified
"""
self.chains = {}
self.locals = defaultdict(list)
self.filename = filename
# deep copy of builtins, to remain reentrant
self._builtins = {k: Def(v) for k, v in Builtins.items()}
# function body are not executed when the function definition is met
# this holds a stack of the functions met during body processing
self._defered = []
# stack of mapping between an id and Names
self._definitions = []
# stack of variable defined with the global keywords
self._promoted_locals = []
# stack of variable that were undefined when we met them, but that may
# be defined in another path of the control flow (esp. in loop)
self._undefs = []
# stack of current node holding definitions: class, module, function...
self._currenthead = []
self._breaks = []
self._continues = []
# dead code levels
self.deadcode = 0
# helpers
def dump_definitions(self, node, ignore_builtins=True):
if isinstance(node, ast.Module) and not ignore_builtins:
builtins = {d for d in self._builtins.values()}
return sorted(d.name()
for d in self.locals[node] if d not in builtins)
else:
return sorted(d.name() for d in self.locals[node])
def dump_chains(self, node):
chains = []
for d in self.locals[node]:
chains.append(str(d))
return chains
def unbound_identifier(self, name, node):
if hasattr(node, "lineno"):
filename = "{}:".format(
"" if self.filename is None else self.filename
)
location = " at {}{}:{}".format(filename,
node.lineno,
node.col_offset)
else:
location = ""
print("W: unbound identifier '{}'{}".format(name, location))
def lookup_identifier(self, name):
for d in reversed(self._definitions):
if name in d:
return d[name]
return []
def defs(self, node):
name = node.id
stars = []
for d in reversed(self._definitions):
if name in d:
return d[name] if not stars else stars + list(d[name])
if "*" in d:
stars.extend(d["*"])
d = self.chains.setdefault(node, Def(node))
if self._undefs:
self._undefs[-1][name].append((d, stars))
if stars:
return stars + [d]
else:
if not self._undefs:
self.unbound_identifier(name, node)
return [d]
def process_body(self, stmts):
deadcode = False
for stmt in stmts:
if isinstance(stmt, (ast.Break, ast.Continue, ast.Raise)):
if not deadcode:
deadcode = True
self.deadcode += 1
self.visit(stmt)
if deadcode:
self.deadcode -= 1
def process_undefs(self):
for undef_name, _undefs in self._undefs[-1].items():
if undef_name in self._definitions[-1]:
for newdef in self._definitions[-1][undef_name]:
for undef, _ in _undefs:
for user in undef.users():
newdef.add_user(user)
else:
for undef, stars in _undefs:
if not stars:
self.unbound_identifier(undef_name, undef.node)
self._undefs.pop()
@contextmanager
def DefinitionContext(self, node):
self._currenthead.append(node)
self._definitions.append(defaultdict(ordered_set))
self._promoted_locals.append(set())
yield
self._promoted_locals.pop()
self._definitions.pop()
self._currenthead.pop()
@contextmanager
def CompDefinitionContext(self, node):
if sys.version_info.major >= 3:
self._currenthead.append(node)
self._definitions.append(defaultdict(ordered_set))
self._promoted_locals.append(set())
yield
if sys.version_info.major >= 3:
self._promoted_locals.pop()
self._definitions.pop()
self._currenthead.pop()
# stmt
def visit_Module(self, node):
self.module = node
with self.DefinitionContext(node):
self._definitions[-1].update(
{k: ordered_set((v,)) for k, v in self._builtins.items()}
)
self._defered.append([])
self.process_body(node.body)
# handle `global' keyword specifically
cg = CollectGlobals()
cg.visit(node)
for nodes in cg.Globals.values():
for n, name in nodes:
if name not in self._definitions[-1]:
dnode = Def((n, name))
self.set_definition(name, dnode)
self.locals[node].append(dnode)
# handle function bodies
for fnode, ctx in self._defered[-1]:
visitor = getattr(self,
"visit_{}".format(type(fnode).__name__))
defs, self._definitions = self._definitions, ctx
visitor(fnode, step=DefinitionStep)
self._definitions = defs
self._defered.pop()
# various sanity checks
if __debug__:
overloaded_builtins = set()
for d in self.locals[node]:
name = d.name()
if name in self._builtins:
overloaded_builtins.add(name)
assert name in self._definitions[0], (name, d.node)
nb_defs = len(self._definitions[0])
nb_bltns = len(self._builtins)
nb_overloaded_bltns = len(overloaded_builtins)
nb_heads = len({d.name() for d in self.locals[node]})
assert nb_defs == nb_heads + nb_bltns - nb_overloaded_bltns
assert not self._definitions
assert not self._defered
def set_definition(self, name, dnode_or_dnodes):
if self.deadcode:
return
if isinstance(dnode_or_dnodes, Def):
self._definitions[-1][name] = ordered_set((dnode_or_dnodes,))
else:
self._definitions[-1][name] = ordered_set(dnode_or_dnodes)
@staticmethod
def add_to_definition(definition, name, dnode_or_dnodes):
if isinstance(dnode_or_dnodes, Def):
definition[name].add(dnode_or_dnodes)
else:
definition[name].update(dnode_or_dnodes)
def extend_definition(self, name, dnode_or_dnodes):
if self.deadcode:
return
DefUseChains.add_to_definition(self._definitions[-1], name,
dnode_or_dnodes)
def visit_FunctionDef(self, node, step=DeclarationStep):
if step is DeclarationStep:
dnode = self.chains.setdefault(node, Def(node))
self.set_definition(node.name, dnode)
self.locals[self._currenthead[-1]].append(dnode)
for kw_default in filter(None, node.args.kw_defaults):
self.visit(kw_default).add_user(dnode)
for default in node.args.defaults:
self.visit(default).add_user(dnode)
for decorator in node.decorator_list:
self.visit(decorator)
definitions = list(self._definitions)
if isinstance(self._currenthead[-1], ast.ClassDef):
definitions.pop()
self._defered[-1].append((node, definitions))
elif step is DefinitionStep:
# function is not considered as defined when evaluating returns
if node.returns:
self.visit(node.returns)
with self.DefinitionContext(node):
self.visit(node.args)
self.process_body(node.body)
else:
raise NotImplementedError()
visit_AsyncFunctionDef = visit_FunctionDef
def visit_ClassDef(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.locals[self._currenthead[-1]].append(dnode)
self.set_definition(node.name, dnode)
for base in node.bases:
self.visit(base).add_user(dnode)
for keyword in node.keywords:
self.visit(keyword.value).add_user(dnode)
for decorator in node.decorator_list:
self.visit(decorator).add_user(dnode)
with self.DefinitionContext(node):
self.set_definition("__class__", Def("__class__"))
self.process_body(node.body)
def visit_Return(self, node):
if node.value:
self.visit(node.value)
def visit_Break(self, _):
for k, v in self._definitions[-1].items():
DefUseChains.add_to_definition(self._breaks[-1], k, v)
self._definitions[-1].clear()
def visit_Continue(self, _):
for k, v in self._definitions[-1].items():
DefUseChains.add_to_definition(self._continues[-1], k, v)
self._definitions[-1].clear()
def visit_Delete(self, node):
for target in node.targets:
self.visit(target)
def visit_Assign(self, node):
# link is implicit through ctx
self.visit(node.value)
for target in node.targets:
self.visit(target)
def visit_AnnAssign(self, node):
if node.value:
dvalue = self.visit(node.value)
dannotation = self.visit(node.annotation)
dtarget = self.visit(node.target)
dtarget.add_user(dannotation)
if node.value:
dvalue.add_user(dtarget)
def visit_AugAssign(self, node):
dvalue = self.visit(node.value)
if isinstance(node.target, ast.Name):
ctx, node.target.ctx = node.target.ctx, ast.Load()
dtarget = self.visit(node.target)
dvalue.add_user(dtarget)
node.target.ctx = ctx
if node.target.id in self._promoted_locals[-1]:
self.extend_definition(node.target.id, dtarget)
else:
loaded_from = [d.name() for d in self.defs(node.target)]
self.set_definition(node.target.id, dtarget)
# If we augassign from a value that comes from '*', let's use
# this node as the definition point.
if '*' in loaded_from:
self.locals[self._currenthead[-1]].append(dtarget)
else:
self.visit(node.target).add_user(dvalue)
def visit_Print(self, node):
if node.dest:
self.visit(node.dest)
for value in node.values:
self.visit(value)
def visit_For(self, node):
self.visit(node.iter)
self._breaks.append(defaultdict(ordered_set))
self._continues.append(defaultdict(ordered_set))
self._undefs.append(defaultdict(list))
self._definitions.append(self._definitions[-1].copy())
self.visit(node.target)
self.process_body(node.body)
self.process_undefs()
continue_defs = self._continues.pop()
for d, u in continue_defs.items():
self.extend_definition(d, u)
self._continues.append(defaultdict(ordered_set))
# extra round to ``emulate'' looping
self.visit(node.target)
self.process_body(node.body)
# process else clause in case of late break
self._definitions.append(defaultdict(ordered_set))
self.process_body(node.orelse)
orelse_defs = self._definitions.pop()
break_defs = self._breaks.pop()
continue_defs = self._continues.pop()
body_defs = self._definitions.pop()
for d, u in orelse_defs.items():
self.extend_definition(d, u)
for d, u in continue_defs.items():
self.extend_definition(d, u)
for d, u in break_defs.items():
self.extend_definition(d, u)
for d, u in body_defs.items():
self.extend_definition(d, u)
visit_AsyncFor = visit_For
def visit_While(self, node):
self._definitions.append(self._definitions[-1].copy())
self._undefs.append(defaultdict(list))
self._breaks.append(defaultdict(ordered_set))
self._continues.append(defaultdict(ordered_set))
self.process_body(node.orelse)
self._definitions.pop()
self._definitions.append(self._definitions[-1].copy())
self.visit(node.test)
self.process_body(node.body)
self.process_undefs()
continue_defs = self._continues.pop()
for d, u in continue_defs.items():
self.extend_definition(d, u)
self._continues.append(defaultdict(ordered_set))
# extra round to simulate loop
self.visit(node.test)
self.process_body(node.body)
# the false branch of the eval
self.visit(node.test)
self._definitions.append(self._definitions[-1].copy())
self.process_body(node.orelse)
orelse_defs = self._definitions.pop()
body_defs = self._definitions.pop()
break_defs = self._breaks.pop()
continue_defs = self._continues.pop()
for d, u in continue_defs.items():
self.extend_definition(d, u)
for d, u in break_defs.items():
self.extend_definition(d, u)
for d, u in orelse_defs.items():
self.extend_definition(d, u)
for d, u in body_defs.items():
self.extend_definition(d, u)
def visit_If(self, node):
self.visit(node.test)
# putting a copy of current level to handle nested conditions
self._definitions.append(self._definitions[-1].copy())
self.process_body(node.body)
body_defs = self._definitions.pop()
self._definitions.append(self._definitions[-1].copy())
self.process_body(node.orelse)
orelse_defs = self._definitions.pop()
for d in body_defs:
if d in orelse_defs:
self.set_definition(d, body_defs[d] + orelse_defs[d])
else:
self.extend_definition(d, body_defs[d])
for d in orelse_defs:
if d in body_defs:
pass # already done in the previous loop
else:
self.extend_definition(d, orelse_defs[d])
def visit_With(self, node):
for withitem in node.items:
self.visit(withitem)
self.process_body(node.body)
visit_AsyncWith = visit_With
def visit_Raise(self, node):
if node.exc:
self.visit(node.exc)
if node.cause:
self.visit(node.cause)
def visit_Try(self, node):
self._definitions.append(self._definitions[-1].copy())
self.process_body(node.body)
self.process_body(node.orelse)
failsafe_defs = self._definitions.pop()
# handle the fact that definitions may have fail
for d in failsafe_defs:
self.extend_definition(d, failsafe_defs[d])
for excepthandler in node.handlers:
self._definitions.append(defaultdict(ordered_set))
self.visit(excepthandler)
handler_def = self._definitions.pop()
for hd in handler_def:
self.extend_definition(hd, handler_def[hd])
self.process_body(node.finalbody)
def visit_Assert(self, node):
self.visit(node.test)
if node.msg:
self.visit(node.msg)
def visit_Import(self, node):
for alias in node.names:
dalias = self.chains.setdefault(alias, Def(alias))
base = alias.name.split(".", 1)[0]
self.set_definition(alias.asname or base, dalias)
self.locals[self._currenthead[-1]].append(dalias)
def visit_ImportFrom(self, node):
for alias in node.names:
dalias = self.chains.setdefault(alias, Def(alias))
self.set_definition(alias.asname or alias.name, dalias)
self.locals[self._currenthead[-1]].append(dalias)
def visit_Exec(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.body)
if node.globals:
self.visit(node.globals)
else:
# any global may be used by this exec!
for defs in self._definitions[0].values():
for d in defs:
d.add_user(dnode)
if node.locals:
self.visit(node.locals)
else:
# any local may be used by this exec!
visible_locals = set()
for _definitions in reversed(self._definitions[1:]):
for dname, defs in _definitions.items():
if dname not in visible_locals:
visible_locals.add(dname)
for d in defs:
d.add_user(dnode)
self.extend_definition("*", dnode)
def visit_Global(self, node):
for name in node.names:
self._promoted_locals[-1].add(name)
def visit_Nonlocal(self, node):
for name in node.names:
for d in reversed(self._definitions[:-1]):
if name not in d:
continue
else:
# this rightfully creates aliasing
self.set_definition(name, d[name])
break
else:
self.unbound_identifier(name, node)
def visit_Expr(self, node):
self.generic_visit(node)
# expr
def visit_BoolOp(self, node):
dnode = self.chains.setdefault(node, Def(node))
for value in node.values:
self.visit(value).add_user(dnode)
return dnode
def visit_BinOp(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.left).add_user(dnode)
self.visit(node.right).add_user(dnode)
return dnode
def visit_UnaryOp(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.operand).add_user(dnode)
return dnode
def visit_Lambda(self, node, step=DeclarationStep):
if step is DeclarationStep:
dnode = self.chains.setdefault(node, Def(node))
self._defered[-1].append((node, list(self._definitions)))
return dnode
elif step is DefinitionStep:
dnode = self.chains[node]
with self.DefinitionContext(node):
self.visit(node.args)
self.visit(node.body).add_user(dnode)
return dnode
else:
raise NotImplementedError()
def visit_IfExp(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.test).add_user(dnode)
self.visit(node.body).add_user(dnode)
self.visit(node.orelse).add_user(dnode)
return dnode
def visit_Dict(self, node):
dnode = self.chains.setdefault(node, Def(node))
for key in filter(None, node.keys):
self.visit(key).add_user(dnode)
for value in node.values:
self.visit(value).add_user(dnode)
return dnode
def visit_Set(self, node):
dnode = self.chains.setdefault(node, Def(node))
for elt in node.elts:
self.visit(elt).add_user(dnode)
return dnode
def visit_ListComp(self, node):
dnode = self.chains.setdefault(node, Def(node))
with self.CompDefinitionContext(node):
for comprehension in node.generators:
self.visit(comprehension).add_user(dnode)
self.visit(node.elt).add_user(dnode)
return dnode
visit_SetComp = visit_ListComp
def visit_DictComp(self, node):
dnode = self.chains.setdefault(node, Def(node))
with self.CompDefinitionContext(node):
for comprehension in node.generators:
self.visit(comprehension).add_user(dnode)
self.visit(node.key).add_user(dnode)
self.visit(node.value).add_user(dnode)
return dnode
visit_GeneratorExp = visit_ListComp
def visit_Await(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.value).add_user(dnode)
return dnode
def visit_Yield(self, node):
dnode = self.chains.setdefault(node, Def(node))
if node.value:
self.visit(node.value).add_user(dnode)
return dnode
visit_YieldFrom = visit_Await
def visit_Compare(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.left).add_user(dnode)
for expr in node.comparators:
self.visit(expr).add_user(dnode)
return dnode
def visit_Call(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.func).add_user(dnode)
for arg in node.args:
self.visit(arg).add_user(dnode)
for kw in node.keywords:
self.visit(kw.value).add_user(dnode)
return dnode
visit_Repr = visit_Await
def visit_Constant(self, node):
dnode = self.chains.setdefault(node, Def(node))
return dnode
def visit_FormattedValue(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.value).add_user(dnode)
if node.format_spec:
self.visit(node.format_spec).add_user(dnode)
return dnode
def visit_JoinedStr(self, node):
dnode = self.chains.setdefault(node, Def(node))
for value in node.values:
self.visit(value).add_user(dnode)
return dnode
visit_Attribute = visit_Await
def visit_Subscript(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.value).add_user(dnode)
self.visit(node.slice).add_user(dnode)
return dnode
visit_Starred = visit_Await
def visit_NamedExpr(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.value).add_user(dnode)
self.visit(node.target)
return dnode
def visit_Name(self, node):
if isinstance(node.ctx, (ast.Param, ast.Store)):
dnode = self.chains.setdefault(node, Def(node))
if node.id in self._promoted_locals[-1]:
self.extend_definition(node.id, dnode)
if dnode not in self.locals[self.module]:
self.locals[self.module].append(dnode)
else:
self.set_definition(node.id, dnode)
if dnode not in self.locals[self._currenthead[-1]]:
self.locals[self._currenthead[-1]].append(dnode)
if node.annotation is not None:
self.visit(node.annotation)
elif isinstance(node.ctx, (ast.Load, ast.Del)):
node_in_chains = node in self.chains
if node_in_chains:
dnode = self.chains[node]
else:
dnode = Def(node)
for d in self.defs(node):
d.add_user(dnode)
if not node_in_chains:
self.chains[node] = dnode
# currently ignore the effect of a del
else:
raise NotImplementedError()
return dnode
def visit_Destructured(self, node):
dnode = self.chains.setdefault(node, Def(node))
tmp_store = ast.Store()
for elt in node.elts:
if isinstance(elt, ast.Name):
tmp_store, elt.ctx = elt.ctx, tmp_store
self.visit(elt)
tmp_store, elt.ctx = elt.ctx, tmp_store
elif isinstance(elt, ast.Subscript):
self.visit(elt)
elif isinstance(elt, (ast.List, ast.Tuple)):
self.visit_Destructured(elt)
return dnode
def visit_List(self, node):
if isinstance(node.ctx, ast.Load):
dnode = self.chains.setdefault(node, Def(node))
for elt in node.elts:
self.visit(elt).add_user(dnode)
return dnode
# unfortunately, destructured node are marked as Load,
# only the parent List/Tuple is marked as Store
elif isinstance(node.ctx, ast.Store):
return self.visit_Destructured(node)
visit_Tuple = visit_List
# slice
def visit_Slice(self, node):
dnode = self.chains.setdefault(node, Def(node))
if node.lower:
self.visit(node.lower).add_user(dnode)
if node.upper:
self.visit(node.upper).add_user(dnode)
if node.step:
self.visit(node.step).add_user(dnode)
return dnode
# misc
def visit_comprehension(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.iter).add_user(dnode)
self.visit(node.target)
for if_ in node.ifs:
self.visit(if_).add_user(dnode)
return dnode
def visit_excepthandler(self, node):
dnode = self.chains.setdefault(node, Def(node))
if node.type:
self.visit(node.type).add_user(dnode)
if node.name:
self.visit(node.name).add_user(dnode)
self.process_body(node.body)
return dnode
def visit_arguments(self, node):
for arg in node.args:
self.visit(arg)
for arg in node.posonlyargs:
self.visit(arg)
if node.vararg:
self.visit(node.vararg)
for arg in node.kwonlyargs:
self.visit(arg)
if node.kwarg:
self.visit(node.kwarg)
def visit_withitem(self, node):
dnode = self.chains.setdefault(node, Def(node))
self.visit(node.context_expr).add_user(dnode)
if node.optional_vars:
self.visit(node.optional_vars)
return dnode
class UseDefChains(object):
"""
DefUseChains adaptor that builds a mapping between each user
and the Def that defines this user:
- chains: Dict[node, List[Def]], a mapping between nodes and the Defs
that define it.
"""
def __init__(self, defuses):
self.chains = {}
for chain in defuses.chains.values():
if isinstance(chain.node, ast.Name):
self.chains.setdefault(chain.node, [])
for use in chain.users():
self.chains.setdefault(use.node, []).append(chain)
for chain in defuses._builtins.values():
for use in chain.users():
self.chains.setdefault(use.node, []).append(chain)
def __str__(self):
out = []
for k, uses in self.chains.items():
kname = Def(k).name()
kstr = "{} <- {{{}}}".format(
kname, ", ".join(sorted(use.name() for use in uses))
)
out.append((kname, kstr))
out.sort()
return ", ".join(s for k, s in out)
if __name__ == "__main__":
import sys
class Beniget(ast.NodeVisitor):
def __init__(self, filename, module):
super(Beniget, self).__init__()
self.filename = filename or ""
self.ancestors = Ancestors()
self.ancestors.visit(module)
self.defuses = DefUseChains(self.filename)
self.defuses.visit(module)
self.visit(module)
def check_unused(self, node, skipped_types=()):
for local_def in self.defuses.locals[node]:
if not local_def.users():
if local_def.name() == "_":
continue # typical naming by-pass
if isinstance(local_def.node, skipped_types):
continue
location = local_def.node
while not hasattr(location, "lineno"):
location = self.ancestors.parent(location)
if isinstance(location, ast.ImportFrom):
if location.module == "__future__":
continue
print(
"W: '{}' is defined but not used at {}:{}:{}".format(
local_def.name(),
self.filename,
location.lineno,
location.col_offset,
)
)
def visit_Module(self, node):
self.generic_visit(node)
if self.filename.endswith("__init__.py"):
return
self.check_unused(
node, skipped_types=(ast.FunctionDef, ast.AsyncFunctionDef,
ast.ClassDef, ast.Name)
)
def visit_FunctionDef(self, node):
self.generic_visit(node)
self.check_unused(node)
paths = sys.argv[1:] or (None,)
for path in paths:
with open(path) if path else sys.stdin as target:
module = ast.parse(target.read())
Beniget(path, module)
beniget-0.4.1/requirements.txt 0000664 0000000 0000000 00000000016 14101703655 0016414 0 ustar 00root root 0000000 0000000 gast ~= 0.5.0
beniget-0.4.1/setup.py 0000664 0000000 0000000 00000002566 14101703655 0014656 0 ustar 00root root 0000000 0000000 try:
from setuptools import setup
kw = {"test_suite": "tests"}
except ImportError:
from distutils.core import setup
kw = {}
setup(
name="beniget", # gast, beniget!
version="0.4.1",
packages=["beniget"],
description="Extract semantic information about static Python code",
long_description="""
A static analyzer for Python2 and Python3 code.
Beniget provides a static over-approximation of the global and
local definitions inside Python Module/Class/Function.
It can also compute def-use chains from each definition.""",
author="serge-sans-paille",
author_email="serge.guelton@telecom-bretagne.eu",
url="https://github.com/serge-sans-paille/beniget/",
license="BSD 3-Clause",
install_requires=open("requirements.txt").read().splitlines(),
classifiers=[
"Development Status :: 4 - Beta",
"Environment :: Console",
"Intended Audience :: Developers",
"License :: OSI Approved :: BSD License",
"Natural Language :: English",
"Programming Language :: Python :: 2",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
],
python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
**kw
)
beniget-0.4.1/tests/ 0000775 0000000 0000000 00000000000 14101703655 0014275 5 ustar 00root root 0000000 0000000 beniget-0.4.1/tests/__init__.py 0000664 0000000 0000000 00000000153 14101703655 0016405 0 ustar 00root root 0000000 0000000 import tests.definitions
import tests.chains
import tests.capture
import tests.attributes
import tests.doc
beniget-0.4.1/tests/attributes.py 0000664 0000000 0000000 00000006066 14101703655 0017045 0 ustar 00root root 0000000 0000000 from unittest import TestCase
from textwrap import dedent
import gast as ast
import beniget
class Attributes(ast.NodeVisitor):
def __init__(self, module_node):
self.chains = beniget.DefUseChains()
self.chains.visit(module_node)
self.attributes = set()
self.users = set()
def visit_ClassDef(self, node):
for stmt in node.body:
if isinstance(stmt, ast.FunctionDef):
self_def = self.chains.chains[stmt.args.args[0]]
self.users.update(use.node for use in self_def.users())
self.generic_visit(node)
def visit_Attribute(self, node):
if node.value in self.users:
self.attributes.add(node.attr)
class TestAttributes(TestCase):
def checkAttribute(self, code, extract, ref):
module = ast.parse(dedent(code))
c = Attributes(module)
c.visit(extract(module))
self.assertEqual(c.attributes, ref)
def test_simple_attribute(self):
code = """
class F:
def bar(self):
return self.bar"""
self.checkAttribute(code, lambda n: n.body[0], {"bar"})
def test_no_attribute(self):
code = """
class F(object):
def bar(self):
return 1"""
self.checkAttribute(code, lambda n: n.body[0], set())
def test_non_standard_self(self):
code = """
class F:
def bar(fels):
return fels.bar + fels.foo"""
self.checkAttribute(code, lambda n: n.body[0], {"bar", "foo"})
def test_self_redefinition(self):
code = """
class F:
def bar(self, other):
self.foo = 1
self = other
return self.bar"""
self.checkAttribute(code, lambda n: n.body[0], {"foo"})
def test_self_redefinition_in_args(self):
code = """
class F:
def bar(self, self):
self.foo = 1"""
self.checkAttribute(code, lambda n: n.body[0], set())
def test_self_redefinition_in_branch_true(self):
code = """
class F:
def bar(self, other):
if other:
self = other
self.foo = 1"""
self.checkAttribute(code, lambda n: n.body[0], {"foo"})
def test_self_redefinition_in_branch_false(self):
code = """
class F:
def bar(self, other):
if not other:
pass
else:
self = other
self.foo = 1"""
self.checkAttribute(code, lambda n: n.body[0], {"foo"})
def test_self_redefinition_in_both_branch(self):
code = """
class F:
def bar(self, other):
if other:
self = other
else:
self = list
return self.pop"""
self.checkAttribute(code, lambda n: n.body[0], set())
beniget-0.4.1/tests/capture.py 0000664 0000000 0000000 00000002500 14101703655 0016307 0 ustar 00root root 0000000 0000000 from unittest import TestCase
from textwrap import dedent
import gast as ast
import beniget
class Capture(ast.NodeVisitor):
def __init__(self, module_node):
self.chains = beniget.DefUseChains()
self.chains.visit(module_node)
self.users = set()
self.captured = set()
def visit_FunctionDef(self, node):
for def_ in self.chains.locals[node]:
self.users.update(use.node for use in def_.users())
self.generic_visit(node)
def visit_Name(self, node):
if isinstance(node.ctx, ast.Load):
if node not in self.users:
# FIXME: IRL, should be the definition of this use
self.captured.add(node.id)
class TestCapture(TestCase):
def checkCapture(self, code, extract, ref):
module = ast.parse(dedent(code))
c = Capture(module)
c.visit(extract(module))
self.assertEqual(c.captured, ref)
def test_simple_capture(self):
code = """
def foo(x):
def bar():
return x"""
self.checkCapture(code, lambda n: n.body[0].body[0], {"x"})
def test_no_capture(self):
code = """
def foo(x):
def bar(x):
return x"""
self.checkCapture(code, lambda n: n.body[0].body[0], set())
beniget-0.4.1/tests/chains.py 0000664 0000000 0000000 00000033616 14101703655 0016125 0 ustar 00root root 0000000 0000000 from contextlib import contextmanager
from unittest import TestCase, skipIf
import gast as ast
import beniget
import io
import sys
@contextmanager
def captured_output():
if sys.version_info.major >= 3:
new_out, new_err = io.StringIO(), io.StringIO()
else:
new_out, new_err = io.BytesIO(), io.BytesIO()
old_out, old_err = sys.stdout, sys.stderr
try:
sys.stdout, sys.stderr = new_out, new_err
yield sys.stdout, sys.stderr
finally:
sys.stdout, sys.stderr = old_out, old_err
class TestDefUseChains(TestCase):
def checkChains(self, code, ref):
class StrictDefUseChains(beniget.DefUseChains):
def unbound_identifier(self, name, node):
raise RuntimeError(
"W: unbound identifier '{}' at {}:{}".format(
name, node.lineno, node.col_offset
)
)
node = ast.parse(code)
c = StrictDefUseChains()
c.visit(node)
self.assertEqual(c.dump_chains(node), ref)
def test_simple_expression(self):
code = "a = 1; a + 2"
self.checkChains(code, ["a -> (a -> (BinOp -> ()))"])
def test_expression_chain(self):
code = "a = 1; (- a + 2) > 0"
self.checkChains(code, ["a -> (a -> (UnaryOp -> (BinOp -> (Compare -> ()))))"])
def test_ifexp_chain(self):
code = "a = 1; a + 1 if a else - a"
self.checkChains(
code,
[
"a -> ("
"a -> (IfExp -> ()), "
"a -> (BinOp -> (IfExp -> ())), "
"a -> (UnaryOp -> (IfExp -> ()))"
")"
],
)
def test_type_destructuring_tuple(self):
code = "a, b = range(2); a"
self.checkChains(code, ["a -> (a -> ())", "b -> ()"])
def test_type_destructuring_list(self):
code = "[a, b] = range(2); a"
self.checkChains(code, ["a -> (a -> ())", "b -> ()"])
def test_type_destructuring_for(self):
code = "for a, b in ((1,2), (3,4)): a"
self.checkChains(code, ["a -> (a -> ())", "b -> ()"])
def test_assign_in_loop(self):
code = "a = 2\nwhile 1: a = 1\na"
self.checkChains(code, ["a -> (a -> ())", "a -> (a -> ())"])
def test_reassign_in_loop(self):
code = "m = 1\nfor i in [1, 2]:\n m = m + 1"
self.checkChains(
code, ["m -> (m -> (BinOp -> ()))", "i -> ()", "m -> (m -> (BinOp -> ()))"]
)
def test_continue_in_loop(self):
code = "for i in [1, 2]:\n if i: m = 1; continue\n m = 1\nm"
self.checkChains(
code, ['i -> (i -> ())', 'm -> (m -> ())', 'm -> (m -> ())']
)
def test_break_in_loop(self):
code = "for i in [1, 2]:\n if i: m = 1; continue\n m = 1\nm"
self.checkChains(
code, ['i -> (i -> ())', 'm -> (m -> ())', 'm -> (m -> ())']
)
def test_augassign(self):
code = "a = 1; a += 2; a"
self.checkChains(code, ['a -> (a -> (a -> ()))'])
def test_expanded_augassign(self):
code = "a = 1; a = a + 2"
self.checkChains(code, ["a -> (a -> (BinOp -> ()))", "a -> ()"])
def test_augassign_in_loop(self):
code = "a = 1\nfor i in [1]:\n a += 2\na"
self.checkChains(code, ['a -> (a -> ((#1), a -> ()), a -> ())',
'i -> ()'])
def test_assign_in_while_in_conditional(self):
code = """
G = 1
while 1:
if 1:
G = 1
G"""
self.checkChains(code, ['G -> (G -> ())',
'G -> (G -> ())'])
def test_assign_in_loop_in_conditional(self):
code = """
G = 1
for _ in [1]:
if 1:
G = 1
G"""
self.checkChains(code, ['G -> (G -> ())',
'_ -> ()',
'G -> (G -> ())'])
def test_simple_print(self):
code = "a = 1; print(a)"
if sys.version_info.major >= 3:
self.checkChains(code, ["a -> (a -> (Call -> ()))"])
else:
self.checkChains(code, ["a -> (a -> ())"])
def test_simple_redefinition(self):
code = "a = 1; a + 2; a = 3; +a"
self.checkChains(
code, ["a -> (a -> (BinOp -> ()))", "a -> (a -> (UnaryOp -> ()))"]
)
def test_simple_for(self):
code = "for i in [1,2,3]: j = i"
self.checkChains(code, ["i -> (i -> ())", "j -> ()"])
def test_simple_for_orelse(self):
code = "for i in [1,2,3]: pass\nelse: i = 4\ni"
self.checkChains(
code,
[
# assign in loop iteration
"i -> (i -> ())",
# assign in orelse
"i -> (i -> ())",
],
)
def test_for_break(self):
code = "i = 8\nfor i in [1,2]:\n break\n i = 3\ni"
self.checkChains(
code,
['i -> (i -> ())',
'i -> (i -> ())',
'i -> ()'])
def test_for_pass(self):
code = "i = 8\nfor i in []:\n pass\ni"
self.checkChains(
code,
['i -> (i -> ())',
'i -> (i -> ())'])
def test_complex_for_orelse(self):
code = "I = J = 0\nfor i in [1,2]:\n if i < 3: I = i\nelse:\n if 1: J = I\nJ"
self.checkChains(
code,
['I -> (I -> ())',
'J -> (J -> ())',
'i -> (i -> (Compare -> ()), i -> ())',
'I -> (I -> ())',
'J -> (J -> ())']
)
def test_simple_while(self):
code = "i = 2\nwhile i: i = i - 1\ni"
self.checkChains(
code,
[
# first assign, out of loop
"i -> (i -> (), i -> (BinOp -> ()), i -> ())",
# second assign, in loop
"i -> (i -> (), i -> (BinOp -> ()), i -> ())",
],
)
def test_while_break(self):
code = "i = 8\nwhile 1:\n break\n i = 3\ni"
self.checkChains(
code,
['i -> (i -> ())',
'i -> ()'])
def test_while_cond_break(self):
code = "i = 8\nwhile 1:\n if i: i=1;break\ni"
self.checkChains(
code,
['i -> (i -> (), i -> ())', 'i -> (i -> ())'])
def test_nested_while(self):
code = '''
done = 1
while done:
while done:
if 1:
done = 1
break
if 1:
break'''
self.checkChains(
code,
['done -> (done -> (), done -> ())',
'done -> (done -> (), done -> ())']
)
def test_while_cond_continue(self):
code = "i = 8\nwhile 1:\n if i: i=1;continue\ni"
self.checkChains(
code,
['i -> (i -> (), i -> ())', 'i -> (i -> (), i -> ())'])
def test_complex_while_orelse(self):
code = "I = J = i = 0\nwhile i:\n if i < 3: I = i\nelse:\n if 1: J = I\nJ"
self.checkChains(
code,
[
"I -> (I -> ())",
"J -> (J -> ())",
"i -> (i -> (), i -> (Compare -> ()), i -> ())",
"J -> (J -> ())",
"I -> (I -> ())",
],
)
def test_while_orelse_break(self):
code = "I = 0\nwhile I:\n if 1: I = 1; break\nelse: I"
self.checkChains(
code,
['I -> (I -> (), I -> ())',
'I -> ()'],
)
def test_while_nested_break(self):
code = "i = 8\nwhile i:\n if i: break\n i = 3\ni"
self.checkChains(
code,
['i -> (i -> (), i -> (), i -> ())',
'i -> (i -> (), i -> (), i -> ())'])
def test_if_true_branch(self):
code = "if 1: i = 0\ni"
self.checkChains(code, ["i -> (i -> ())"])
def test_if_false_branch(self):
code = "if 1: pass\nelse: i = 0\ni"
self.checkChains(code, ["i -> (i -> ())"])
def test_if_both_branch(self):
code = "if 1: i = 1\nelse: i = 0\ni"
self.checkChains(code, ["i -> (i -> ())"] * 2)
def test_if_in_loop(self):
code = "for _ in [0, 1]:\n if _: i = 1\n else: j = i\ni"
self.checkChains(code, ["_ -> (_ -> ())", "i -> (i -> (), i -> ())", "j -> ()"])
def test_with_handler(self):
code = 'with open("/dev/null") as x: pass\nx'
self.checkChains(code, ["x -> (x -> ())"])
def test_simple_try(self):
code = 'try: e = open("/dev/null")\nexcept Exception: pass\ne'
self.checkChains(code, ["e -> (e -> ())"])
def test_simple_except(self):
code = "try: pass\nexcept Exception as e: pass\ne"
self.checkChains(code, ["e -> (e -> ())"])
def test_simple_try_except(self):
code = 'try: f = open("")\nexcept Exception as e: pass\ne;f'
self.checkChains(code, ["f -> (f -> ())", "e -> (e -> ())"])
def test_redef_try_except(self):
code = 'try: f = open("")\nexcept Exception as f: pass\nf'
self.checkChains(code, ["f -> (f -> ())", "f -> (f -> ())"])
def test_simple_import(self):
code = "import x; x"
self.checkChains(code, ["x -> (x -> ())"])
def test_simple_import_as(self):
code = "import x as y; y()"
self.checkChains(code, ["y -> (y -> (Call -> ()))"])
def test_multiple_import_as(self):
code = "import x as y, z; y"
self.checkChains(code, ["y -> (y -> ())", "z -> ()"])
def test_import_from(self):
code = "from y import x; x"
self.checkChains(code, ["x -> (x -> ())"])
def test_import_from_as(self):
code = "from y import x as z; z"
self.checkChains(code, ["z -> (z -> ())"])
def test_multiple_import_from_as(self):
code = "from y import x as z, w; z"
self.checkChains(code, ["z -> (z -> ())", "w -> ()"])
def test_method_function_conflict(self):
code = "def foo():pass\nclass C:\n def foo(self): foo()"
self.checkChains(code, ["foo -> (foo -> (Call -> ()))", "C -> ()"])
def test_nested_if(self):
code = "f = 1\nif 1:\n if 1:pass\n else: f=1\nelse: f = 1\nf"
self.checkChains(code, ["f -> (f -> ())", "f -> (f -> ())", "f -> (f -> ())"])
def test_nested_if_else(self):
code = "f = 1\nif 1: f = 1\nelse:\n if 1:pass\n else: f=1\nf"
self.checkChains(code, ["f -> (f -> ())", "f -> (f -> ())", "f -> (f -> ())"])
def test_try_except(self):
code = "f = 1\ntry: \n len(); f = 2\nexcept: pass\nf"
self.checkChains(code, ["f -> (f -> ())", "f -> (f -> ())"])
def test_attr(self):
code = "import numpy as bar\ndef foo():\n return bar.zeros(2)"
self.checkChains(
code, ["bar -> (bar -> (Attribute -> (Call -> ())))", "foo -> ()"]
)
def test_class_decorator(self):
code = "from some import decorator\n@decorator\nclass C:pass"
self.checkChains(code, ["decorator -> (decorator -> (C -> ()))", "C -> ()"])
@skipIf(sys.version_info.major < 3, "Python 3 syntax")
def test_functiondef_returns(self):
code = "x = 1\ndef foo() -> x: pass"
self.checkChains(code, ['x -> (x -> ())', 'foo -> ()'])
@skipIf(sys.version_info.major < 3, "Python 3 syntax")
def test_class_annotation(self):
code = "type_ = int\ndef foo(bar: type_): pass"
self.checkChains(code, ["type_ -> (type_ -> ())", "foo -> ()"])
def check_unbound_identifier_message(self, code, expected_messages, filename=None):
node = ast.parse(code)
c = beniget.DefUseChains(filename)
with captured_output() as (out, err):
c.visit(node)
produced_messages = out.getvalue().strip().split("\n")
self.assertEqual(len(expected_messages), len(produced_messages))
for expected, produced in zip(expected_messages, produced_messages):
self.assertIn(expected, produced, "actual message contains expected message")
def test_unbound_identifier_message_format(self):
code = "foo(1)\nbar(2)"
self.check_unbound_identifier_message(code, [":1", ":2"])
self.check_unbound_identifier_message(code, ["foo.py:1", "foo.py:2"], filename="foo.py")
def test_star_import_with_conditional_redef(self):
code = '''
from math import *
if 1:
def pop():
cos()
cos = pop()'''
self.checkChains(code, [
'* -> (cos -> (Call -> ()))',
'pop -> (pop -> (Call -> ()))',
'cos -> (cos -> (Call -> ()))'
])
@skipIf(sys.version_info < (3, 8), 'Python 3.8 syntax')
def test_named_expr_simple(self):
code = '''
if (x := 1):
y = x + 1'''
self.checkChains(
code, ['x -> (x -> (BinOp -> ()))', 'y -> ()']
)
@skipIf(sys.version_info < (3, 8), 'Python 3.8 syntax')
def test_named_expr_complex(self):
code = '''
if (x := (y := 1) + 1):
z = x + y'''
self.checkChains(
code, ['y -> (y -> (BinOp -> ()))', 'x -> (x -> (BinOp -> ()))', 'z -> ()']
)
@skipIf(sys.version_info < (3, 8), 'Python 3.8 syntax')
def test_named_expr_with_rename(self):
code = '''
a = 1
if (a := a + a):
pass'''
self.checkChains(
code, ['a -> (a -> (BinOp -> (NamedExpr -> ())), a -> (BinOp -> (NamedExpr -> ())))', 'a -> ()']
)
class TestUseDefChains(TestCase):
def checkChains(self, code, ref):
class StrictDefUseChains(beniget.DefUseChains):
def unbound_identifier(self, name, node):
raise RuntimeError(
"W: unbound identifier '{}' at {}:{}".format(
name, node.lineno, node.col_offset
)
)
node = ast.parse(code)
c = StrictDefUseChains()
c.visit(node)
cc = beniget.UseDefChains(c)
self.assertEqual(str(cc), ref)
def test_simple_expression(self):
code = "a = 1; a"
self.checkChains(code, "a <- {a}, a <- {}")
def test_call(self):
code = "from foo import bar; bar(1, 2)"
self.checkChains(code, "Call <- {Constant, Constant, bar}, bar <- {bar}")
beniget-0.4.1/tests/definitions.py 0000664 0000000 0000000 00000026740 14101703655 0017173 0 ustar 00root root 0000000 0000000 from unittest import TestCase
import gast as ast
import beniget
import sys
class StrictDefUseChains(beniget.DefUseChains):
def unbound_identifier(self, name, node):
raise RuntimeError(
"W: unbound identifier '{}' at {}:{}".format(
name, node.lineno, node.col_offset
)
)
class TestGlobals(TestCase):
def checkGlobals(self, code, ref):
node = ast.parse(code)
c = StrictDefUseChains()
c.visit(node)
self.assertEqual(c.dump_definitions(node), ref)
def test_SingleFunctionDef(self):
code = "def foo(): pass"
self.checkGlobals(code, ["foo"])
def test_MultipleFunctionDef(self):
code = "def foo(): pass\ndef bar(): return"
self.checkGlobals(code, ["bar", "foo"])
def testFuntionRedefinion(self):
code = "def foo(): pass\ndef foo(): return"
self.checkGlobals(code, ["foo", "foo"])
def testFuntionNested(self):
code = "def foo():\n def bar(): return"
self.checkGlobals(code, ["foo"])
if sys.version_info.major >= 3:
def testAsyncFunctionDef(self):
code = "async def foo(): pass"
self.checkGlobals(code, ["foo"])
def testClassDef(self):
code = "class C:pass"
self.checkGlobals(code, ["C"])
def testDelClassDef(self):
code = "class C:pass\ndel C"
self.checkGlobals(code, ["C"])
def testDelClassDefReDef(self):
code = "class C:pass\ndel C\nclass C:pass"
self.checkGlobals(code, ["C", "C"])
def testNestedClassDef(self):
code = "class C:\n class D: pass"
self.checkGlobals(code, ["C"])
def testMultipleClassDef(self):
code = "class C: pass\nclass D: pass"
self.checkGlobals(code, ["C", "D"])
def testClassRedefinition(self):
code = "class C: pass\nclass C: pass"
self.checkGlobals(code, ["C", "C"])
def testClassMethodDef(self):
code = "class C:\n def some(self):pass"
self.checkGlobals(code, ["C"])
def testGlobalDef(self):
code = "x = 1"
self.checkGlobals(code, ["x"])
if sys.version_info.major >= 3:
def testGlobalAnnotatedDef(self):
code = "x : 1"
self.checkGlobals(code, ["x"])
def testMultipleGlobalDef(self):
code = "x = 1; x = 2"
self.checkGlobals(code, ["x", "x"])
def testGlobalDestructuring(self):
code = "x, y = 1, 2"
self.checkGlobals(code, ["x", "y"])
def testGlobalAugAssign(self):
code = "x = 1; x += 2"
self.checkGlobals(code, ["x"])
def testGlobalFor(self):
code = "for x in (1,2): pass"
self.checkGlobals(code, ["x"])
def testGlobalForDestructuring(self):
code = "for x, y in [(1,2)]: pass"
self.checkGlobals(code, ["x", "y"])
def testGlobalNestedFor(self):
code = "for x in (1,2):\n for y in (2, 1): pass"
self.checkGlobals(code, ["x", "y"])
def testGlobalInFor(self):
code = "for x in (1,2): y = x"
self.checkGlobals(code, ["x", "y"])
if sys.version_info >= (3, 7):
def testGlobalAsyncFor(self):
code = "async for x in (1,2): pass"
self.checkGlobals(code, ["x"])
def testGlobalInWhile(self):
code = "while True: x = 1"
self.checkGlobals(code, ["x"])
def testGlobalInIfTrueBranch(self):
code = "if 1: a = 1"
self.checkGlobals(code, ["a"])
def testGlobalInIfFalseBranch(self):
code = "if 1: pass\nelse: a = 1"
self.checkGlobals(code, ["a"])
def testGlobalInIfBothBranch(self):
code = "if 1: a = 1\nelse: a = 2"
self.checkGlobals(code, ["a", "a"])
def testGlobalInIfBothBranchDifferent(self):
code = "if 1: a = 1\nelse: b = 2"
self.checkGlobals(code, ["a", "b"])
def testGlobalWith(self):
code = "from some import foo\nwith foo() as x: pass"
self.checkGlobals(code, ["foo", "x"])
if sys.version_info >= (3, 7):
def testGlobalAsyncWith(self):
code = "from some import foo\nasync with foo() as x: pass"
self.checkGlobals(code, ["foo", "x"])
def testGlobalTry(self):
code = "try: x = 1\nexcept Exception: pass"
self.checkGlobals(code, ["x"])
def testGlobalTryExcept(self):
code = "from some import foo\ntry: foo()\nexcept Exception as e: pass"
self.checkGlobals(code, ["e", "foo"])
def testGlobalTryExceptFinally(self):
code = "try: w = 1\nexcept Exception as x: y = 1\nfinally: z = 1"
self.checkGlobals(code, ["w", "x", "y", "z"])
def testGlobalThroughKeyword(self):
code = "def foo(): global x"
self.checkGlobals(code, ["foo", "x"])
def testGlobalThroughKeywords(self):
code = "def foo(): global x, y"
self.checkGlobals(code, ["foo", "x", "y"])
def testGlobalThroughMultipleKeyword(self):
code = "def foo(): global x\ndef bar(): global x"
self.checkGlobals(code, ["bar", "foo", "x"])
def testGlobalBeforeKeyword(self):
code = "x = 1\ndef foo(): global x"
self.checkGlobals(code, ["foo", "x"])
def testGlobalsBeforeKeyword(self):
code = "x = 1\ndef foo(): global x, y"
self.checkGlobals(code, ["foo", "x", "y"])
if sys.version_info.major >= 3:
def testGlobalAfterKeyword(self):
code = "def foo(): global x\nx : 1"
self.checkGlobals(code, ["foo", "x"])
def testGlobalsAfterKeyword(self):
code = "def foo(): global x, y\ny : 1"
self.checkGlobals(code, ["foo", "x", "y"])
def testGlobalImport(self):
code = "import foo"
self.checkGlobals(code, ["foo"])
def testGlobalImports(self):
code = "import foo, bar"
self.checkGlobals(code, ["bar", "foo"])
def testGlobalImportSubModule(self):
code = "import foo.bar"
self.checkGlobals(code, ["foo"])
def testGlobalImportSubModuleAs(self):
code = "import foo.bar as foobar"
self.checkGlobals(code, ["foobar"])
def testGlobalImportAs(self):
code = "import foo as bar"
self.checkGlobals(code, ["bar"])
def testGlobalImportsAs(self):
code = "import foo as bar, foobar"
self.checkGlobals(code, ["bar", "foobar"])
def testGlobalImportFrom(self):
code = "from foo import bar"
self.checkGlobals(code, ["bar"])
def testGlobalImportFromAs(self):
code = "from foo import bar as BAR"
self.checkGlobals(code, ["BAR"])
def testGlobalImportFromStar(self):
code = "from foo import *"
self.checkGlobals(code, ["*"])
def testGlobalImportFromStarRedefine(self):
code = "from foo import *\nx+=1"
self.checkGlobals(code, ["*", "x"])
def testGlobalImportsFrom(self):
code = "from foo import bar, man"
self.checkGlobals(code, ["bar", "man"])
def testGlobalImportsFromAs(self):
code = "from foo import bar, man as maid"
self.checkGlobals(code, ["bar", "maid"])
def testGlobalListComp(self):
code = "from some import y; [1 for x in y]"
if sys.version_info.major == 2:
self.checkGlobals(code, ["x", "y"])
else:
self.checkGlobals(code, ["y"])
def testGlobalSetComp(self):
code = "from some import y; {1 for x in y}"
if sys.version_info.major == 2:
self.checkGlobals(code, ["x", "y"])
else:
self.checkGlobals(code, ["y"])
def testGlobalDictComp(self):
code = "from some import y; {1:1 for x in y}"
if sys.version_info.major == 2:
self.checkGlobals(code, ["x", "y"])
else:
self.checkGlobals(code, ["y"])
def testGlobalGeneratorExpr(self):
code = "from some import y; (1 for x in y)"
if sys.version_info.major == 2:
self.checkGlobals(code, ["x", "y"])
else:
self.checkGlobals(code, ["y"])
def testGlobalLambda(self):
code = "lambda x: x"
self.checkGlobals(code, [])
class TestClasses(TestCase):
def checkClasses(self, code, ref):
node = ast.parse(code)
c = StrictDefUseChains()
c.visit(node)
classes = [n for n in node.body if isinstance(n, ast.ClassDef)]
assert len(classes) == 1, "only one top-level function per test case"
cls = classes[0]
self.assertEqual(c.dump_definitions(cls), ref)
def test_class_method_assign(self):
code = "class C:\n def foo(self):pass\n bar = foo"
self.checkClasses(code, ["bar", "foo"])
class TestLocals(TestCase):
def checkLocals(self, code, ref):
node = ast.parse(code)
c = StrictDefUseChains()
c.visit(node)
functions = [n for n in node.body if isinstance(n, ast.FunctionDef)]
assert len(functions) == 1, "only one top-level function per test case"
f = functions[0]
self.assertEqual(c.dump_definitions(f), ref)
def testLocalFunctionDef(self):
code = "def foo(): pass"
self.checkLocals(code, [])
def testLocalFunctionDefOneArg(self):
code = "def foo(a): pass"
self.checkLocals(code, ["a"])
def testLocalFunctionDefOneArgDefault(self):
code = "def foo(a=1): pass"
self.checkLocals(code, ["a"])
def testLocalFunctionDefArgsDefault(self):
code = "def foo(a, b=1): pass"
self.checkLocals(code, ["a", "b"])
def testLocalFunctionDefStarArgs(self):
code = "def foo(a, *b): pass"
self.checkLocals(code, ["a", "b"])
def testLocalFunctionDefKwArgs(self):
code = "def foo(a, **b): pass"
self.checkLocals(code, ["a", "b"])
if sys.version_info.major >= 3:
def testLocalFunctionDefKwOnly(self):
code = "def foo(a, *, b=1): pass"
self.checkLocals(code, ["a", "b"])
if sys.version_info.major == 2:
def testLocalFunctionDefDestructureArg(self):
code = "def foo((a, b)): pass"
self.checkLocals(code, ["a", "b"])
def test_LocalAssign(self):
code = "def foo(): a = 1"
self.checkLocals(code, ["a"])
def test_LocalAssignRedef(self):
code = "def foo(a): a = 1"
self.checkLocals(code, ["a", "a"])
def test_LocalNestedFun(self):
code = "def foo(a):\n def bar(): return a\n return bar"
self.checkLocals(code, ["a", "bar"])
if sys.version_info.major >= 3:
def test_LocalNonLocalBefore(self):
code = "def foo(a):\n def bar():\n nonlocal a; a = 1\n bar(); return a"
self.checkLocals(code, ["a", "bar"])
def test_LocalNonLocalAfter(self):
code = (
"def foo():\n def bar():\n nonlocal a; a = 1\n a = 2; bar(); return a"
)
self.checkLocals(code, ["a", "bar"])
def test_LocalGlobal(self):
code = "def foo(): global a; a = 1"
self.checkLocals(code, [])
def test_ListCompInLoop(self):
code = "def foo(i):\n for j in i:\n [k for k in j]"
if sys.version_info.major == 2:
self.checkLocals(code, ["i", "j", "k"])
else:
self.checkLocals(code, ["i", "j"])
def test_AugAssignInLoop(self):
code = """
def foo(X, f):
for i in range(2):
if i == 0: A = f * X[:, i]
else: A += f * X[:, i]
return A"""
self.checkLocals(code, ["A", "X", "f", "i"])
def test_IfInWhile(self):
code = """
def foo(a):
while(a):
if a == 1: print(b)
else: b = a"""
self.checkLocals(code, ["a", "b"])
beniget-0.4.1/tests/doc.py 0000664 0000000 0000000 00000000563 14101703655 0015420 0 ustar 00root root 0000000 0000000 import doctest
import unittest
import os
import beniget
class TestDoctest(unittest.TestCase):
def test_beniget_documentation(self):
failed, _ = doctest.testmod(beniget.beniget)
self.assertEqual(failed, 0)
def test_beniget_readme(self):
failed, _ = doctest.testfile(os.path.join("..", "README.rst"))
self.assertEqual(failed, 0)
beniget-0.4.1/tox.ini 0000664 0000000 0000000 00000000134 14101703655 0014444 0 ustar 00root root 0000000 0000000 [tox]
envlist = py27,py36
[testenv]
deps = -rrequirements.txt
commands=python setup.py test