././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1635411751.439627 pip_check_reqs-2.3.2/0000755000076500000240000000000000000000000013263 5ustar00adamstaff././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1635411751.435837 pip_check_reqs-2.3.2/.github/0000755000076500000240000000000000000000000014623 5ustar00adamstaff././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1635411751.4376085 pip_check_reqs-2.3.2/.github/workflows/0000755000076500000240000000000000000000000016660 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1627314031.0 pip_check_reqs-2.3.2/.github/workflows/ci.yml0000644000076500000240000000362000000000000017777 0ustar00adamstaff--- name: CI on: push: branches: [master] pull_request: branches: [master] schedule: # * is a special character in YAML so you have to quote this string # Run at 1:00 every day - cron: '0 1 * * *' jobs: build: runs-on: ubuntu-latest strategy: matrix: # These versions match the minimum and maximum versions of pip in # requirements.txt. # An empty string here represents the latest version. pip-version: ['==10.0.1', ''] python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - name: "Set up Python" uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 with: path: ~/.cache/pip # This is like the example but we use ``*requirements.txt`` rather # than ``requirements.txt`` because we have multiple requirements # files. key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt') }} restore-keys: | ${{ runner.os }}-pip- - name: "Install dependencies" run: | python -m pip install --upgrade 'pip ${{ matrix.pip-version }}' # We use '--ignore-installed' to avoid GitHub's cache which can cause # issues - we have seen packages from this cache be cause trouble with # pip-extra-reqs. python -m pip install --ignore-installed --upgrade --editable .[dev] python -m pip install flake8 - name: "Lint" run: | flake8 *.py pip_check_reqs tests pip-extra-reqs pip_check_reqs pip-missing-reqs pip_check_reqs - name: "Run tests" run: | pytest -s -vvv --cov-fail-under 100 --cov=pip_check_reqs/ --cov=tests tests/ --cov-report=xml - name: "Upload coverage to Codecov" uses: "codecov/codecov-action@v1" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615034025.0 pip_check_reqs-2.3.2/.gitignore0000644000076500000240000000104000000000000015246 0ustar00adamstaff# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ bin/ build/ develop-eggs/ dist/ eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .cache nosetests.xml coverage.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject # Rope .ropeproject # Django stuff: *.log *.pot # Sphinx documentation docs/_build/ ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411718.0 pip_check_reqs-2.3.2/CHANGELOG.rst0000644000076500000240000000517700000000000015316 0ustar00adamstaff Release History --------------- 2.3.2 - Fixed support for pip < 21.3 2.3.1 - Fixed `--skip-incompatible` skipping other requirements too. - Support pip >= 21.3 2.3.0 - Support pip >= 21.2.1 2.2.2 - AST parsing failures will now report tracebacks with a proper filename for the parsed frame, instead of ``. 2.2.1 - Python source is now always read using utf-8, even if default encoding for reading files is set otherwise. 2.2.0 - Added `--skip-incompatible` flag to `pip-extra-reqs`, which makes it ignore requirements with environment markers that are incompatible with the current environment. - Added `--requirements-file` flag to `pip-extra-reqs` and `pip-missing-reqs` commands. This flag makes it possible to specify a path to the requirements file. Previously, `"requirements.txt"` was always used. - Fixed some of the logs not being visible with `-d` and `-v` flags. 2.1.1 - Bug fix: Though Python 2 support was removed from the source code, the published wheel was still universal. The published wheel now explicitly does not support Python 2. Please use version 2.0.4 for Python 2. 2.1.0 - Remove support for Python 2. Please use an older version of this tool if you require that support. - Remove requirement for setuptools. - Support newer versions of pip, including the current version, for more features (20.1.1). Thanks to @Czaki for important parts of this change. 2.0.1 - handled removal of normalize_name from pip.utils - handle packages with no files 2.0 **renamed package to pip_check_reqs** - added tool pip-extra-reqs to find packages installed but not used (contributed by Josh Hesketh) 1.2.1 - relax requirement to 6.0+ 1.2.0 - bumped pip requirement to 6.0.8+ - updated use of pip internals to match that version 1.1.9 - test fixes and cleanup - remove hard-coded simplejson debugging behaviour 1.1.8 - use os.path.realpath to avoid symlink craziness on debian/ubuntu 1.1.7 - tweak to debug output 1.1.6 - add debug (very verbose) run output 1.1.5 - add header to output to make it clearer when in a larger test run - fix tests and self-test 1.1.4 - add --version - remove debug print from released code lol 1.1.3 - fix program to generate exit code useful for testing 1.1.2 - corrected version of vendored search_packages_info() from pip - handle relative imports 1.1.1 - fixed handling of import from __future__ - self-tested and added own requirements.txt - cleaned up usage to require a file or directory to scan (rather than defaulting to ".") - vendored code from pip 1.6dev which fixes bug in search_packages_info until pip 1.6 is released 1.1.0 - implemented --ignore-module ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615034025.0 pip_check_reqs-2.3.2/CONTRIBUTING.rst0000644000076500000240000000076300000000000015732 0ustar00adamstaffContributing ============ Release process --------------- * Update the CHANGELOG on the master branch * Update ``__version__`` in ``pip_check_reqs/__init__.py`` on the master branch. Run the following steps: .. code:: sh git checkout master && \ git pull && \ pip install twine && \ pip install -r requirements.txt && \ rm -rf build dist && \ git status # There should be no uncommitted changes. && \ python setup.py sdist bdist_wheel && \ twine upload -r pypi dist/* ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615034025.0 pip_check_reqs-2.3.2/LICENSE0000644000076500000240000000207000000000000014267 0ustar00adamstaffThe MIT License (MIT) Copyright (c) 2015 Richard Jones Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615034025.0 pip_check_reqs-2.3.2/MANIFEST.in0000644000076500000240000000024700000000000015024 0ustar00adamstaffrecursive-include pip_check_reqs include *.rst include LICENSE include setup.* include tox.ini include requirements.txt include test-requirements.txt include MANIFEST ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1635411751.439689 pip_check_reqs-2.3.2/PKG-INFO0000644000076500000240000001554000000000000014365 0ustar00adamstaffMetadata-Version: 2.1 Name: pip_check_reqs Version: 2.3.2 Summary: Find packages that should or should not be in requirements for a project Home-page: https://github.com/r1chardj0n3s/pip-check-reqs Author: Richard Jones Author-email: r1chardj0n3s@gmail.com Maintainer: Adam Dangoor Maintainer-email: adamdangoor@gmail.com License: MIT Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Build Tools Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3 Provides-Extra: dev License-File: LICENSE |Build Status| |codecov| |PyPI| .. |Build Status| image:: https://github.com/r1chardj0n3s/pip-check-reqs/workflows/CI/badge.svg :target: https://github.com/r1chardj0n3s/pip-check-reqs/actions .. |codecov| image:: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs/branch/master/graph/badge.svg :target: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs .. |PyPI| image:: https://badge.fury.io/py/pip-check-reqs.svg :target: https://badge.fury.io/py/pip-check-reqs pip-check-reqs ============== It happens: you start using a module in your project and it works and you don't realise that it's only being included in your `virtualenv`_ because it's a dependency of a package you're using. pip-missing-reqs finds those modules so you can include them in the `requirements.txt`_ for the project. Alternatively, you have a long-running project that has some packages in requirements.txt that are no longer actively used in the codebase. The pip-extra-reqs tool will find those modules so you can remove them. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ .. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files Assuming your project follows a layout like the suggested `sample project`_:: setup.py setup.cfg requirements.txt sample/__init__.py sample/sample.py sample/tests/test_sample.py .. _`sample project`: https://packaging.python.org/en/latest/tutorial.html#creating-your-own-project Basic usage, running in your project directory:: pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample This will find all imports in the code in "sample" and check that the packages those modules belong to are in the requirements.txt file. Additionally it is possible to check that there are no dependencies in requirements.txt that are then unused in the project:: pip-extra-reqs --ignore-file=sample/tests/* sample This would find anything that is listed in requirements.txt but that is not imported by sample. Sample tox.ini configuration ---------------------------- To make your life easier, copy something like this into your tox.ini:: [testenv:pip-check-reqs] deps=-rrequirements.txt commands= pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample Excluding test files (or others) from this check ------------------------------------------------ Your test files will sometimes be present in the same directory as your application source ("sample" in the above examples). The requirements for those tests generally should not be in the requirements.txt file, and you don't want this tool to generate false hits for those. You may exclude those test files from your check using the `--ignore-file` option (shorthand is `-f`). Multiple instances of the option are allowed. Excluding modules from the check -------------------------------- If your project has modules which are conditionally imported, or requirements which are conditionally included, you may exclude certain modules from the check by name (or glob pattern) using `--ignore-module` (shorthand is `-m`):: # ignore the module spam pip-missing-reqs --ignore-module=spam sample # ignore the whole package spam as well pip-missing-reqs --ignore-module=spam --ignore-module=spam.* sample With Thanks To -------------- Josh Hesketh -- who refactored code and contributed the pip-extra-reqs tool. Wil Cooley -- who handled the removal of normalize_name and fixed some bugs. Release History --------------- 2.3.2 - Fixed support for pip < 21.3 2.3.1 - Fixed `--skip-incompatible` skipping other requirements too. - Support pip >= 21.3 2.3.0 - Support pip >= 21.2.1 2.2.2 - AST parsing failures will now report tracebacks with a proper filename for the parsed frame, instead of ``. 2.2.1 - Python source is now always read using utf-8, even if default encoding for reading files is set otherwise. 2.2.0 - Added `--skip-incompatible` flag to `pip-extra-reqs`, which makes it ignore requirements with environment markers that are incompatible with the current environment. - Added `--requirements-file` flag to `pip-extra-reqs` and `pip-missing-reqs` commands. This flag makes it possible to specify a path to the requirements file. Previously, `"requirements.txt"` was always used. - Fixed some of the logs not being visible with `-d` and `-v` flags. 2.1.1 - Bug fix: Though Python 2 support was removed from the source code, the published wheel was still universal. The published wheel now explicitly does not support Python 2. Please use version 2.0.4 for Python 2. 2.1.0 - Remove support for Python 2. Please use an older version of this tool if you require that support. - Remove requirement for setuptools. - Support newer versions of pip, including the current version, for more features (20.1.1). Thanks to @Czaki for important parts of this change. 2.0.1 - handled removal of normalize_name from pip.utils - handle packages with no files 2.0 **renamed package to pip_check_reqs** - added tool pip-extra-reqs to find packages installed but not used (contributed by Josh Hesketh) 1.2.1 - relax requirement to 6.0+ 1.2.0 - bumped pip requirement to 6.0.8+ - updated use of pip internals to match that version 1.1.9 - test fixes and cleanup - remove hard-coded simplejson debugging behaviour 1.1.8 - use os.path.realpath to avoid symlink craziness on debian/ubuntu 1.1.7 - tweak to debug output 1.1.6 - add debug (very verbose) run output 1.1.5 - add header to output to make it clearer when in a larger test run - fix tests and self-test 1.1.4 - add --version - remove debug print from released code lol 1.1.3 - fix program to generate exit code useful for testing 1.1.2 - corrected version of vendored search_packages_info() from pip - handle relative imports 1.1.1 - fixed handling of import from __future__ - self-tested and added own requirements.txt - cleaned up usage to require a file or directory to scan (rather than defaulting to ".") - vendored code from pip 1.6dev which fixes bug in search_packages_info until pip 1.6 is released 1.1.0 - implemented --ignore-module ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1622279512.0 pip_check_reqs-2.3.2/README.rst0000644000076500000240000000712300000000000014755 0ustar00adamstaff|Build Status| |codecov| |PyPI| .. |Build Status| image:: https://github.com/r1chardj0n3s/pip-check-reqs/workflows/CI/badge.svg :target: https://github.com/r1chardj0n3s/pip-check-reqs/actions .. |codecov| image:: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs/branch/master/graph/badge.svg :target: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs .. |PyPI| image:: https://badge.fury.io/py/pip-check-reqs.svg :target: https://badge.fury.io/py/pip-check-reqs pip-check-reqs ============== It happens: you start using a module in your project and it works and you don't realise that it's only being included in your `virtualenv`_ because it's a dependency of a package you're using. pip-missing-reqs finds those modules so you can include them in the `requirements.txt`_ for the project. Alternatively, you have a long-running project that has some packages in requirements.txt that are no longer actively used in the codebase. The pip-extra-reqs tool will find those modules so you can remove them. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ .. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files Assuming your project follows a layout like the suggested `sample project`_:: setup.py setup.cfg requirements.txt sample/__init__.py sample/sample.py sample/tests/test_sample.py .. _`sample project`: https://packaging.python.org/en/latest/tutorial.html#creating-your-own-project Basic usage, running in your project directory:: pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample This will find all imports in the code in "sample" and check that the packages those modules belong to are in the requirements.txt file. Additionally it is possible to check that there are no dependencies in requirements.txt that are then unused in the project:: pip-extra-reqs --ignore-file=sample/tests/* sample This would find anything that is listed in requirements.txt but that is not imported by sample. Sample tox.ini configuration ---------------------------- To make your life easier, copy something like this into your tox.ini:: [testenv:pip-check-reqs] deps=-rrequirements.txt commands= pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample Excluding test files (or others) from this check ------------------------------------------------ Your test files will sometimes be present in the same directory as your application source ("sample" in the above examples). The requirements for those tests generally should not be in the requirements.txt file, and you don't want this tool to generate false hits for those. You may exclude those test files from your check using the `--ignore-file` option (shorthand is `-f`). Multiple instances of the option are allowed. Excluding modules from the check -------------------------------- If your project has modules which are conditionally imported, or requirements which are conditionally included, you may exclude certain modules from the check by name (or glob pattern) using `--ignore-module` (shorthand is `-m`):: # ignore the module spam pip-missing-reqs --ignore-module=spam sample # ignore the whole package spam as well pip-missing-reqs --ignore-module=spam --ignore-module=spam.* sample With Thanks To -------------- Josh Hesketh -- who refactored code and contributed the pip-extra-reqs tool. Wil Cooley -- who handled the removal of normalize_name and fixed some bugs. ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1635411751.4382665 pip_check_reqs-2.3.2/pip_check_reqs/0000755000076500000240000000000000000000000016242 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411718.0 pip_check_reqs-2.3.2/pip_check_reqs/__init__.py0000644000076500000240000000002600000000000020351 0ustar00adamstaff__version__ = '2.3.2' ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411718.0 pip_check_reqs-2.3.2/pip_check_reqs/common.py0000644000076500000240000002053300000000000020107 0ustar00adamstaffimport ast import fnmatch import imp import logging import os import re from typing import Container, Optional, List, cast from packaging.utils import canonicalize_name from packaging.markers import Marker # Between different versions of pip the location of PipSession has changed. try: from pip._internal.network.session import PipSession except ImportError: # pragma: no cover from pip._internal.download import PipSession from pip._internal.req.req_file import parse_requirements try: from pip._internal.utils.misc import get_installed_distributions except ImportError: # pip>=21.3 from pip._internal.utils.compat import stdlib_pkgs from pip._internal.metadata import get_default_environment, get_environment from pip._internal.metadata.pkg_resources import Distribution as _Dist from pip._vendor.pkg_resources import Distribution # get_installed_distributions was removed in pip 21.3. # This is a copy from pip. # See # https://github.com/pypa/pip/commit/d051a00fc57037104fca85ad8ebf2cdbd1e32d24#diff-058e40cb3a9ea705f655937e48f3a053f5dc7c500b7f1b2aae76e9bd673faf64. # # This is mocked in all tests (unfortunately) and so we do not cover this # function. def get_installed_distributions( local_only: bool = True, skip: Container[str] = stdlib_pkgs, include_editables: bool = True, editables_only: bool = False, user_only: bool = False, paths: Optional[List[str]] = None, ) -> List[Distribution]: # pragma: no cover """Return a list of installed Distribution objects. Left for compatibility until direct pkg_resources uses are refactored out. """ if paths is None: env = get_default_environment() else: env = get_environment(paths) dists = env.iter_installed_distributions( local_only=local_only, skip=skip, include_editables=include_editables, editables_only=editables_only, user_only=user_only, ) return [cast(_Dist, dist)._dist for dist in dists] log = logging.getLogger(__name__) class FoundModule: def __init__(self, modname, filename, locations=None): self.modname = modname self.filename = os.path.realpath(filename) self.locations = locations or [] # filename, lineno def __repr__(self): return 'FoundModule("%s")' % self.modname class ImportVisitor(ast.NodeVisitor): def __init__(self, options): super(ImportVisitor, self).__init__() self.__options = options self.__modules = {} self.__location = None def set_location(self, location): self.__location = location def visit_Import(self, node): for alias in node.names: self.__addModule(alias.name, node.lineno) def visit_ImportFrom(self, node): if node.module == '__future__': # not an actual module return for alias in node.names: if node.module is None: # relative import continue self.__addModule(node.module + '.' + alias.name, node.lineno) def __addModule(self, modname, lineno): if self.__options.ignore_mods(modname): return path = None progress = [] modpath = last_modpath = None for p in modname.split('.'): try: file, modpath, description = imp.find_module(p, path) except ImportError: # the component specified at this point is not importable # (is just an attr of the module) # *or* it's not actually installed, so we don't care either break # success! we found *something* progress.append(p) # we might have previously seen a useful path though... if modpath is None: # pragma: no cover # the sys module will hit this code path on py3k - possibly # others will, but I've not discovered them modpath = last_modpath break # ... though it might not be a file, so not interesting to us if not os.path.isdir(modpath): break path = [modpath] last_modpath = modpath if modpath is None: # the module doesn't actually appear to exist on disk return modname = '.'.join(progress) if modname not in self.__modules: self.__modules[modname] = FoundModule(modname, modpath) self.__modules[modname].locations.append((self.__location, lineno)) def finalise(self): return self.__modules def pyfiles(root): d = os.path.abspath(root) if not os.path.isdir(d): n, ext = os.path.splitext(d) if ext == '.py': yield d else: raise ValueError('%s is not a python file or directory' % root) for root, dirs, files in os.walk(d): for f in files: n, ext = os.path.splitext(f) if ext == '.py': yield os.path.join(root, f) def find_imported_modules(options): vis = ImportVisitor(options) for path in options.paths: for filename in pyfiles(path): if options.ignore_files(filename): log.info('ignoring: %s', os.path.relpath(filename)) continue log.debug('scanning: %s', os.path.relpath(filename)) with open(filename, encoding='utf-8') as f: content = f.read() vis.set_location(filename) vis.visit(ast.parse(content, filename)) return vis.finalise() def find_required_modules(options, requirements_filename: str): explicit = set() for requirement in parse_requirements(requirements_filename, session=PipSession()): try: requirement_name = requirement.name # The type of "requirement" changed between pip versions. # We exclude the "except" from coverage so that on any pip version we # can report 100% coverage. except AttributeError: # pragma: no cover from pip._internal.req.constructors import install_req_from_line requirement_name = install_req_from_line( requirement.requirement, ).name if options.ignore_reqs(requirement): log.debug('ignoring requirement: %s', requirement_name) continue if options.skip_incompatible: requirement_string = requirement.requirement if not has_compatible_markers(requirement_string): log.debug('ignoring requirement (incompatible environment ' 'marker): %s', requirement_string) continue log.debug('found requirement: %s', requirement_name) explicit.add(canonicalize_name(requirement_name)) return explicit def has_compatible_markers(full_requirement: str) -> bool: if ';' not in full_requirement: return True # No environment marker. enviroment_marker = full_requirement.split(';')[1] if not enviroment_marker: return True # Empty environment marker. return Marker(enviroment_marker).evaluate() def is_package_file(path): '''Determines whether the path points to a Python package sentinel file - the __init__.py or its compiled variants. ''' m = re.search(r'(.+)/__init__\.py[co]?$', path) if m is not None: return m.group(1) return '' def ignorer(ignore_cfg): if not ignore_cfg: return lambda candidate: False def f(candidate, ignore_cfg=ignore_cfg): for ignore in ignore_cfg: try: from pip._internal.req.constructors import ( install_req_from_line, ) candidate_path = install_req_from_line( # pragma: no cover candidate.requirement, ).name except (ImportError, AttributeError): try: candidate_path = candidate.name except AttributeError: candidate_path = candidate if fnmatch.fnmatch(candidate_path, ignore): return True elif fnmatch.fnmatch(os.path.relpath(candidate_path), ignore): return True return False return f ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1634719762.0 pip_check_reqs-2.3.2/pip_check_reqs/find_extra_reqs.py0000644000076500000240000001512500000000000021775 0ustar00adamstaffimport collections import logging import pathlib import optparse import os import sys from packaging.utils import canonicalize_name from pip._internal.commands.show import search_packages_info from pip_check_reqs import common from pip_check_reqs.common import get_installed_distributions log = logging.getLogger(__name__) def find_extra_reqs(options, requirements_filename): # 1. find files used by imports in the code (as best we can without # executing) used_modules = common.find_imported_modules(options) # 2. find which packages provide which files installed_files = {} all_pkgs = (pkg.project_name for pkg in get_installed_distributions()) for package in search_packages_info(all_pkgs): if isinstance(package, dict): # pragma: no cover package_name = package['name'] package_location = package['location'] package_files = package.get('files', []) or [] else: # pragma: no cover package_name = package.name package_location = package.location package_files = [] for item in (package.files or []): here = pathlib.Path('.').resolve() item_location_rel = (pathlib.Path(package_location) / item) item_location = item_location_rel.resolve() try: relative_item_location = item_location.relative_to(here) except ValueError: # Ideally we would use Pathlib.is_relative_to rather than # checking for a ValueError, but that is only available in # Python 3.9+. relative_item_location = item_location package_files.append(str(relative_item_location)) log.debug('installed package: %s (at %s)', package_name, package_location) for package_file in package_files: path = os.path.realpath( os.path.join(package_location, package_file), ) installed_files[path] = package_name package_path = common.is_package_file(path) if package_path: # we've seen a package file so add the bare package directory # to the installed list as well as we might want to look up # a package by its directory path later installed_files[package_path] = package_name # 3. match imported modules against those packages used = collections.defaultdict(list) for modname, info in used_modules.items(): # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) log.debug('used module: %s (from package %s)', modname, installed_files[info.filename]) used[used_name].append(info) else: log.debug( 'used module: %s (from file %s, assuming stdlib or local)', modname, info.filename) # 4. compare with requirements explicit = common.find_required_modules( options=options, requirements_filename=requirements_filename, ) return [name for name in explicit if name not in used] def main(): from pip_check_reqs import __version__ usage = 'usage: %prog [options] files or directories' parser = optparse.OptionParser(usage) parser.add_option("--requirements-file", dest="requirements_filename", metavar="PATH", default="requirements.txt", help="path to the requirements file " "(defaults to \"requirements.txt\")") parser.add_option("-f", "--ignore-file", dest="ignore_files", action="append", default=[], help="file paths globs to ignore") parser.add_option("-m", "--ignore-module", dest="ignore_mods", action="append", default=[], help="used module names (globs are ok) to ignore") parser.add_option("-r", "--ignore-requirement", dest="ignore_reqs", action="append", default=[], help="reqs in requirements to ignore") parser.add_option("-s", "--skip-incompatible", dest="skip_incompatible", action="store_true", default=False, help="skip requirements that have incompatible " "environment markers") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="be more verbose") parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="be *really* verbose") parser.add_option("--version", dest="version", action="store_true", default=False, help="display version information") (options, args) = parser.parse_args() if options.version: sys.exit(__version__) if not args: parser.error("no source files or directories specified") sys.exit(2) options.ignore_files = common.ignorer(options.ignore_files) options.ignore_mods = common.ignorer(options.ignore_mods) options.ignore_reqs = common.ignorer(options.ignore_reqs) options.paths = args logging.basicConfig(format='%(message)s') if options.debug: level = logging.DEBUG elif options.verbose: level = logging.INFO else: level = logging.WARN log.setLevel(level) common.log.setLevel(level) log.info('using pip_check_reqs-%s from %s', __version__, __file__) extras = find_extra_reqs( options=options, requirements_filename=options.requirements_filename, ) if extras: log.warning('Extra requirements:') for name in extras: message = '{name} in {requirements_filename}'.format( name=name, requirements_filename=options.requirements_filename, ) log.warning(message) if extras: sys.exit(1) if __name__ == '__main__': # pragma: no cover main() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1634719762.0 pip_check_reqs-2.3.2/pip_check_reqs/find_missing_reqs.py0000644000076500000240000001565100000000000022327 0ustar00adamstaffimport collections import logging import optparse import os import pathlib import sys from packaging.utils import canonicalize_name from pip._internal.commands.show import search_packages_info # Between different versions of pip the location of PipSession has changed. try: from pip._internal.network.session import PipSession except ImportError: # pragma: no cover from pip._internal.download import PipSession from pip._internal.req.req_file import parse_requirements from pip_check_reqs import common from pip_check_reqs.common import get_installed_distributions log = logging.getLogger(__name__) def find_missing_reqs(options, requirements_filename): # 1. find files used by imports in the code (as best we can without # executing) used_modules = common.find_imported_modules(options) # 2. find which packages provide which files installed_files = {} all_pkgs = (pkg.project_name for pkg in get_installed_distributions()) for package in search_packages_info(all_pkgs): if isinstance(package, dict): # pragma: no cover package_name = package['name'] package_location = package['location'] package_files = package.get('files', []) or [] else: # pragma: no cover package_name = package.name package_location = package.location package_files = [] for item in (package.files or []): here = pathlib.Path('.').resolve() item_location_rel = (pathlib.Path(package_location) / item) item_location = item_location_rel.resolve() try: relative_item_location = item_location.relative_to(here) except ValueError: # Ideally we would use Pathlib.is_relative_to rather than # checking for a ValueError, but that is only available in # Python 3.9+. relative_item_location = item_location package_files.append(str(relative_item_location)) log.debug('installed package: %s (at %s)', package_name, package_location) for package_file in package_files: path = os.path.realpath( os.path.join(package_location, package_file), ) installed_files[path] = package_name package_path = common.is_package_file(path) if package_path: # we've seen a package file so add the bare package directory # to the installed list as well as we might want to look up # a package by its directory path later installed_files[package_path] = package_name # 3. match imported modules against those packages used = collections.defaultdict(list) for modname, info in used_modules.items(): # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) log.debug('used module: %s (from package %s)', modname, installed_files[info.filename]) used[used_name].append(info) else: log.debug( 'used module: %s (from file %s, assuming stdlib or local)', modname, info.filename) # 4. compare with requirements explicit = set() for requirement in parse_requirements( requirements_filename, session=PipSession(), ): try: requirement_name = requirement.name # The type of "requirement" changed between pip versions. # We exclude the "except" from coverage so that on any pip version we # can report 100% coverage. except AttributeError: # pragma: no cover from pip._internal.req.constructors import install_req_from_line requirement_name = install_req_from_line( requirement.requirement, ).name log.debug('found requirement: %s', requirement_name) explicit.add(canonicalize_name(requirement_name)) return [(name, used[name]) for name in used if name not in explicit] def main(): from pip_check_reqs import __version__ usage = 'usage: %prog [options] files or directories' parser = optparse.OptionParser(usage) parser.add_option("--requirements-file", dest="requirements_filename", metavar="PATH", default="requirements.txt", help="path to the requirements file " "(defaults to \"requirements.txt\")") parser.add_option("-f", "--ignore-file", dest="ignore_files", action="append", default=[], help="file paths globs to ignore") parser.add_option("-m", "--ignore-module", dest="ignore_mods", action="append", default=[], help="used module names (globs are ok) to ignore") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, help="be more verbose") parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="be *really* verbose") parser.add_option("--version", dest="version", action="store_true", default=False, help="display version information") (options, args) = parser.parse_args() if options.version: sys.exit(__version__) if not args: parser.error("no source files or directories specified") sys.exit(2) options.ignore_files = common.ignorer(options.ignore_files) options.ignore_mods = common.ignorer(options.ignore_mods) options.paths = args logging.basicConfig(format='%(message)s') if options.debug: level = logging.DEBUG elif options.verbose: level = logging.INFO else: level = logging.WARN log.setLevel(level) common.log.setLevel(level) log.info('using pip_check_reqs-%s from %s', __version__, __file__) missing = find_missing_reqs( options=options, requirements_filename=options.requirements_filename, ) if missing: log.warning('Missing requirements:') for name, uses in missing: for use in uses: for filename, lineno in use.locations: log.warning('%s:%s dist=%s module=%s', os.path.relpath(filename), lineno, name, use.modname) if missing: sys.exit(1) if __name__ == '__main__': # pragma: no cover main() ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1635411751.4389622 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/0000755000076500000240000000000000000000000017734 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411751.0 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/PKG-INFO0000644000076500000240000001554000000000000021036 0ustar00adamstaffMetadata-Version: 2.1 Name: pip-check-reqs Version: 2.3.2 Summary: Find packages that should or should not be in requirements for a project Home-page: https://github.com/r1chardj0n3s/pip-check-reqs Author: Richard Jones Author-email: r1chardj0n3s@gmail.com Maintainer: Adam Dangoor Maintainer-email: adamdangoor@gmail.com License: MIT Platform: UNKNOWN Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Build Tools Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3 Provides-Extra: dev License-File: LICENSE |Build Status| |codecov| |PyPI| .. |Build Status| image:: https://github.com/r1chardj0n3s/pip-check-reqs/workflows/CI/badge.svg :target: https://github.com/r1chardj0n3s/pip-check-reqs/actions .. |codecov| image:: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs/branch/master/graph/badge.svg :target: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs .. |PyPI| image:: https://badge.fury.io/py/pip-check-reqs.svg :target: https://badge.fury.io/py/pip-check-reqs pip-check-reqs ============== It happens: you start using a module in your project and it works and you don't realise that it's only being included in your `virtualenv`_ because it's a dependency of a package you're using. pip-missing-reqs finds those modules so you can include them in the `requirements.txt`_ for the project. Alternatively, you have a long-running project that has some packages in requirements.txt that are no longer actively used in the codebase. The pip-extra-reqs tool will find those modules so you can remove them. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ .. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files Assuming your project follows a layout like the suggested `sample project`_:: setup.py setup.cfg requirements.txt sample/__init__.py sample/sample.py sample/tests/test_sample.py .. _`sample project`: https://packaging.python.org/en/latest/tutorial.html#creating-your-own-project Basic usage, running in your project directory:: pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample This will find all imports in the code in "sample" and check that the packages those modules belong to are in the requirements.txt file. Additionally it is possible to check that there are no dependencies in requirements.txt that are then unused in the project:: pip-extra-reqs --ignore-file=sample/tests/* sample This would find anything that is listed in requirements.txt but that is not imported by sample. Sample tox.ini configuration ---------------------------- To make your life easier, copy something like this into your tox.ini:: [testenv:pip-check-reqs] deps=-rrequirements.txt commands= pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample Excluding test files (or others) from this check ------------------------------------------------ Your test files will sometimes be present in the same directory as your application source ("sample" in the above examples). The requirements for those tests generally should not be in the requirements.txt file, and you don't want this tool to generate false hits for those. You may exclude those test files from your check using the `--ignore-file` option (shorthand is `-f`). Multiple instances of the option are allowed. Excluding modules from the check -------------------------------- If your project has modules which are conditionally imported, or requirements which are conditionally included, you may exclude certain modules from the check by name (or glob pattern) using `--ignore-module` (shorthand is `-m`):: # ignore the module spam pip-missing-reqs --ignore-module=spam sample # ignore the whole package spam as well pip-missing-reqs --ignore-module=spam --ignore-module=spam.* sample With Thanks To -------------- Josh Hesketh -- who refactored code and contributed the pip-extra-reqs tool. Wil Cooley -- who handled the removal of normalize_name and fixed some bugs. Release History --------------- 2.3.2 - Fixed support for pip < 21.3 2.3.1 - Fixed `--skip-incompatible` skipping other requirements too. - Support pip >= 21.3 2.3.0 - Support pip >= 21.2.1 2.2.2 - AST parsing failures will now report tracebacks with a proper filename for the parsed frame, instead of ``. 2.2.1 - Python source is now always read using utf-8, even if default encoding for reading files is set otherwise. 2.2.0 - Added `--skip-incompatible` flag to `pip-extra-reqs`, which makes it ignore requirements with environment markers that are incompatible with the current environment. - Added `--requirements-file` flag to `pip-extra-reqs` and `pip-missing-reqs` commands. This flag makes it possible to specify a path to the requirements file. Previously, `"requirements.txt"` was always used. - Fixed some of the logs not being visible with `-d` and `-v` flags. 2.1.1 - Bug fix: Though Python 2 support was removed from the source code, the published wheel was still universal. The published wheel now explicitly does not support Python 2. Please use version 2.0.4 for Python 2. 2.1.0 - Remove support for Python 2. Please use an older version of this tool if you require that support. - Remove requirement for setuptools. - Support newer versions of pip, including the current version, for more features (20.1.1). Thanks to @Czaki for important parts of this change. 2.0.1 - handled removal of normalize_name from pip.utils - handle packages with no files 2.0 **renamed package to pip_check_reqs** - added tool pip-extra-reqs to find packages installed but not used (contributed by Josh Hesketh) 1.2.1 - relax requirement to 6.0+ 1.2.0 - bumped pip requirement to 6.0.8+ - updated use of pip internals to match that version 1.1.9 - test fixes and cleanup - remove hard-coded simplejson debugging behaviour 1.1.8 - use os.path.realpath to avoid symlink craziness on debian/ubuntu 1.1.7 - tweak to debug output 1.1.6 - add debug (very verbose) run output 1.1.5 - add header to output to make it clearer when in a larger test run - fix tests and self-test 1.1.4 - add --version - remove debug print from released code lol 1.1.3 - fix program to generate exit code useful for testing 1.1.2 - corrected version of vendored search_packages_info() from pip - handle relative imports 1.1.1 - fixed handling of import from __future__ - self-tested and added own requirements.txt - cleaned up usage to require a file or directory to scan (rather than defaulting to ".") - vendored code from pip 1.6dev which fixes bug in search_packages_info until pip 1.6 is released 1.1.0 - implemented --ignore-module ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411751.0 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/SOURCES.txt0000644000076500000240000000111600000000000021617 0ustar00adamstaff.gitignore CHANGELOG.rst CONTRIBUTING.rst LICENSE MANIFEST.in README.rst requirements.txt setup.cfg setup.py test-requirements.txt .github/workflows/ci.yml pip_check_reqs/__init__.py pip_check_reqs/common.py pip_check_reqs/find_extra_reqs.py pip_check_reqs/find_missing_reqs.py pip_check_reqs.egg-info/PKG-INFO pip_check_reqs.egg-info/SOURCES.txt pip_check_reqs.egg-info/dependency_links.txt pip_check_reqs.egg-info/entry_points.txt pip_check_reqs.egg-info/requires.txt pip_check_reqs.egg-info/top_level.txt tests/test_common.py tests/test_find_extra_reqs.py tests/test_find_missing_reqs.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411751.0 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/dependency_links.txt0000644000076500000240000000000100000000000024002 0ustar00adamstaff ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411751.0 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/entry_points.txt0000644000076500000240000000020100000000000023223 0ustar00adamstaff[console_scripts] pip-extra-reqs = pip_check_reqs.find_extra_reqs:main pip-missing-reqs = pip_check_reqs.find_missing_reqs:main ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411751.0 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/requires.txt0000644000076500000240000000007500000000000022336 0ustar00adamstaffpackaging>=16.0 pip>=10.0.1 [dev] pretend pytest pytest-cov ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1635411751.0 pip_check_reqs-2.3.2/pip_check_reqs.egg-info/top_level.txt0000644000076500000240000000001700000000000022464 0ustar00adamstaffpip_check_reqs ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615238208.0 pip_check_reqs-2.3.2/requirements.txt0000644000076500000240000000014500000000000016547 0ustar00adamstaffpackaging >= 16.0 # Pinned pip versions are matched in the GitHub workflows CI matrix. pip >= 10.0.1 ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1635411751.4398592 pip_check_reqs-2.3.2/setup.cfg0000644000076500000240000000004600000000000015104 0ustar00adamstaff[egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615034025.0 pip_check_reqs-2.3.2/setup.py0000644000076500000240000000362700000000000015005 0ustar00adamstafffrom setuptools import setup from codecs import open from os import path from pathlib import Path from typing import List from pip_check_reqs import __version__ here = path.abspath(path.dirname(__file__)) def _get_dependencies(requirements_file: Path) -> List[str]: """ Return requirements from a requirements file. This expects a requirements file with no ``--find-links`` lines. """ lines = requirements_file.read_text().strip().split('\n') return [line for line in lines if not line.startswith('#')] with open(path.join(here, 'README.rst'), encoding='utf-8') as f: long_description = f.read() with open(path.join(here, 'CHANGELOG.rst'), encoding='utf-8') as f: long_description += f.read() INSTALL_REQUIRES = _get_dependencies( requirements_file=Path('requirements.txt'), ) DEV_REQUIRES = _get_dependencies( requirements_file=Path('test-requirements.txt'), ) setup( name='pip_check_reqs', version=__version__, description=( 'Find packages that should or should not be in requirements for a ' 'project'), long_description=long_description, url='https://github.com/r1chardj0n3s/pip-check-reqs', author='Richard Jones', author_email='r1chardj0n3s@gmail.com', maintainer='Adam Dangoor', maintainer_email='adamdangoor@gmail.com', license='MIT', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Topic :: Software Development :: Build Tools', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3', ], packages=['pip_check_reqs'], entry_points={ 'console_scripts': [ 'pip-missing-reqs=pip_check_reqs.find_missing_reqs:main', 'pip-extra-reqs=pip_check_reqs.find_extra_reqs:main', ], }, install_requires=INSTALL_REQUIRES, extras_require={'dev': DEV_REQUIRES}, ) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615034025.0 pip_check_reqs-2.3.2/test-requirements.txt0000644000076500000240000000003200000000000017517 0ustar00adamstaffpretend pytest pytest-cov ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1635411751.4394412 pip_check_reqs-2.3.2/tests/0000755000076500000240000000000000000000000014425 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1634719762.0 pip_check_reqs-2.3.2/tests/test_common.py0000644000076500000240000001525300000000000017334 0ustar00adamstafffrom __future__ import absolute_import import ast import logging import os.path from pathlib import Path import pytest import pretend from pip_check_reqs import common @pytest.mark.parametrize( ["path", "result"], [ ('/', ''), ('__init__.py', ''), # a top-level file like this has no package name ('/__init__.py', ''), # no package name ('spam/__init__.py', 'spam'), ('spam/__init__.pyc', 'spam'), ('spam/__init__.pyo', 'spam'), ('ham/spam/__init__.py', 'ham/spam'), ('/ham/spam/__init__.py', '/ham/spam'), ]) def test_is_package_file(path, result): assert common.is_package_file(path) == result def test_FoundModule(): fm = common.FoundModule('spam', 'ham') assert fm.modname == 'spam' assert fm.filename == os.path.realpath('ham') assert fm.locations == [] assert str(fm) == 'FoundModule("spam")' @pytest.mark.parametrize( ["stmt", "result"], [ ('import ast', ['ast']), ('import ast, sys', ['ast', 'sys']), ('from sys import version', ['sys']), ('from os import path', ['os']), ('import distutils.command.check', ['distutils']), ('import spam', []), # don't break because bad programmer ]) def test_ImportVisitor(stmt, result): class options: def ignore_mods(self, modname): return False vis = common.ImportVisitor(options()) vis.set_location('spam.py') vis.visit(ast.parse(stmt)) result = vis.finalise() assert set(result.keys()) == set(result) def test_pyfiles_file(monkeypatch): monkeypatch.setattr(os.path, 'abspath', pretend.call_recorder(lambda x: '/spam/ham.py')) assert list(common.pyfiles('spam')) == ['/spam/ham.py'] def test_pyfiles_file_no_dice(monkeypatch): monkeypatch.setattr(os.path, 'abspath', pretend.call_recorder(lambda x: '/spam/ham')) with pytest.raises(ValueError): list(common.pyfiles('spam')) def test_pyfiles_package(monkeypatch): monkeypatch.setattr(os.path, 'abspath', pretend.call_recorder(lambda x: '/spam')) monkeypatch.setattr(os.path, 'isdir', pretend.call_recorder(lambda x: True)) walk_results = [ ('spam', [], ['__init__.py', 'spam', 'ham.py']), ('spam/dub', [], ['bass.py', 'dropped']), ] monkeypatch.setattr(os, 'walk', pretend.call_recorder(lambda x: walk_results)) assert list(common.pyfiles('spam')) == \ ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py'] @pytest.mark.parametrize(["ignore_ham", "ignore_hashlib", "expect", "locs"], [ (False, False, ['ast', 'os', 'hashlib'], [('spam.py', 2), ('ham.py', 2)]), (False, True, ['ast', 'os'], [('spam.py', 2), ('ham.py', 2)]), (True, False, ['ast'], [('spam.py', 2)]), (True, True, ['ast'], [('spam.py', 2)]), ]) def test_find_imported_modules(monkeypatch, caplog, ignore_ham, ignore_hashlib, expect, locs): monkeypatch.setattr(common, 'pyfiles', pretend.call_recorder(lambda x: ['spam.py', 'ham.py'])) class FakeFile(): contents = [ 'from os import path\nimport ast, hashlib', 'from __future__ import braces\nimport ast, sys\n' 'from . import friend', ] def __init__(self, filename, encoding=None): pass def read(self): return self.contents.pop() def __enter__(self): return self def __exit__(self, *args): pass monkeypatch.setattr(common, 'open', FakeFile, raising=False) caplog.set_level(logging.INFO) class options: paths = ['dummy'] verbose = True @staticmethod def ignore_files(path): if path == 'ham.py' and ignore_ham: return True return False @staticmethod def ignore_mods(module): if module == 'hashlib' and ignore_hashlib: return True return False result = common.find_imported_modules(options) assert set(result) == set(expect) assert result['ast'].locations == locs if ignore_ham: assert caplog.records[0].message == 'ignoring: ham.py' @pytest.mark.parametrize(["ignore_cfg", "candidate", "result"], [ ([], 'spam', False), ([], 'ham', False), (['spam'], 'spam', True), (['spam'], 'spam.ham', False), (['spam'], 'eggs', False), (['spam*'], 'spam', True), (['spam*'], 'spam.ham', True), (['spam*'], 'eggs', False), (['spam'], '/spam', True), ]) def test_ignorer(monkeypatch, tmp_path: Path, ignore_cfg, candidate, result): monkeypatch.setattr(os.path, 'relpath', lambda s: s.lstrip('/')) ignorer = common.ignorer(ignore_cfg) assert ignorer(candidate) == result def test_find_required_modules(monkeypatch, tmp_path: Path): class options: skip_incompatible = False options.ignore_reqs = common.ignorer(ignore_cfg=['barfoo']) fake_requirements_file = tmp_path / 'requirements.txt' fake_requirements_file.write_text('foobar==1\nbarfoo==2') reqs = common.find_required_modules( options=options, requirements_filename=str(fake_requirements_file), ) assert reqs == set(['foobar']) def test_find_required_modules_env_markers(monkeypatch, tmp_path): class options: skip_incompatible = True def ignore_reqs(self, modname): return False fake_requirements_file = tmp_path / 'requirements.txt' fake_requirements_file.write_text('spam==1; python_version<"2.0"\n' 'ham==2;\n' 'eggs==3\n') reqs = common.find_required_modules( options=options(), requirements_filename=str(fake_requirements_file), ) assert reqs == {'ham', 'eggs'} def test_find_imported_modules_sets_encoding_to_utf8_when_reading(tmp_path): (tmp_path / 'module.py').touch() class options: paths = [tmp_path] def ignore_files(*_): return False expected_encoding = 'utf-8' used_encoding = None original_open = common.__builtins__['open'] def mocked_open(*args, **kwargs): # As of Python 3.9, the args to open() are as follows: # file, mode, buffering, encoding, erorrs, newline, closedf, opener nonlocal used_encoding if 'encoding' in kwargs: used_encoding = kwargs['encoding'] return original_open(*args, **kwargs) common.__builtins__['open'] = mocked_open common.find_imported_modules(options) common.__builtins__['open'] = original_open assert used_encoding == expected_encoding ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615238208.0 pip_check_reqs-2.3.2/tests/test_find_extra_reqs.py0000644000076500000240000001234400000000000021217 0ustar00adamstafffrom __future__ import absolute_import import collections import logging import optparse from pathlib import Path import pytest import pretend from pip_check_reqs import find_extra_reqs, common, __version__ @pytest.fixture def fake_opts(): class FakeOptParse: class options: requirements_filename = 'requirements.txt' paths = ['dummy'] verbose = False debug = False version = False ignore_files = [] ignore_mods = [] ignore_reqs = [] options = options() args = ['ham.py'] def __init__(self, usage): pass def add_option(*args, **kw): pass def parse_args(self): return (self.options, self.args) return FakeOptParse def test_find_extra_reqs(monkeypatch, tmp_path: Path): imported_modules = dict(spam=common.FoundModule('spam', 'site-spam/spam.py', [('ham.py', 1)]), shrub=common.FoundModule('shrub', 'site-spam/shrub.py', [('ham.py', 3)]), ignore=common.FoundModule('ignore', 'ignore.py', [('ham.py', 2)])) monkeypatch.setattr(common, 'find_imported_modules', pretend.call_recorder(lambda a: imported_modules)) FakeDist = collections.namedtuple('FakeDist', ['project_name']) installed_distributions = map(FakeDist, ['spam', 'pass']) monkeypatch.setattr(find_extra_reqs, 'get_installed_distributions', pretend.call_recorder(lambda: installed_distributions)) packages_info = [ dict(name='spam', location='site-spam', files=['spam/__init__.py', 'spam/shrub.py']), dict(name='shrub', location='site-spam', files=['shrub.py']), dict(name='pass', location='site-spam', files=['pass.py']), ] monkeypatch.setattr(find_extra_reqs, 'search_packages_info', pretend.call_recorder(lambda x: packages_info)) fake_requirements_file = tmp_path / 'requirements.txt' fake_requirements_file.write_text('foobar==1') class options: def ignore_reqs(x, y): return False skip_incompatible = False options = options() result = find_extra_reqs.find_extra_reqs( options=options, requirements_filename=str(fake_requirements_file), ) assert result == ['foobar'] def test_main_failure(monkeypatch, caplog, fake_opts): monkeypatch.setattr(optparse, 'OptionParser', fake_opts) caplog.set_level(logging.WARN) monkeypatch.setattr(find_extra_reqs, 'find_extra_reqs', lambda options, requirements_filename: ['extra']) with pytest.raises(SystemExit) as excinfo: find_extra_reqs.main() assert excinfo.value.code == 1 assert caplog.records[0].message == \ 'Extra requirements:' assert caplog.records[1].message == \ 'extra in requirements.txt' def test_main_no_spec(monkeypatch, caplog, fake_opts): fake_opts.args = [] monkeypatch.setattr(optparse, 'OptionParser', fake_opts) monkeypatch.setattr(fake_opts, 'error', pretend.call_recorder(lambda s, e: None), raising=False) with pytest.raises(SystemExit) as excinfo: find_extra_reqs.main() assert excinfo.value.code == 2 assert fake_opts.error.calls @pytest.mark.parametrize(["verbose_cfg", "debug_cfg", "result"], [ (False, False, ['warn']), (True, False, ['info', 'warn']), (False, True, ['debug', 'info', 'warn']), (True, True, ['debug', 'info', 'warn']), ]) def test_logging_config(monkeypatch, caplog, verbose_cfg, debug_cfg, result): class options: requirements_filename = '' paths = ['dummy'] verbose = verbose_cfg debug = debug_cfg version = False ignore_files = [] ignore_mods = [] ignore_reqs = [] options = options() class FakeOptParse: def __init__(self, usage): pass def add_option(*args, **kw): pass def parse_args(self): return (options, ['ham.py']) monkeypatch.setattr(optparse, 'OptionParser', FakeOptParse) monkeypatch.setattr( find_extra_reqs, 'find_extra_reqs', lambda options, requirements_filename: [], ) find_extra_reqs.main() for event in [(logging.DEBUG, 'debug'), (logging.INFO, 'info'), (logging.WARN, 'warn')]: find_extra_reqs.log.log(*event) messages = [r.message for r in caplog.records] # first message is always the usage message if verbose_cfg or debug_cfg: assert messages[1:] == result else: assert messages == result def test_main_version(monkeypatch, caplog, fake_opts): fake_opts.options.version = True monkeypatch.setattr(optparse, 'OptionParser', fake_opts) with pytest.raises(SystemExit) as excinfo: find_extra_reqs.main() assert str(excinfo.value) == __version__ ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1615238208.0 pip_check_reqs-2.3.2/tests/test_find_missing_reqs.py0000644000076500000240000001273200000000000021546 0ustar00adamstafffrom __future__ import absolute_import import collections import logging import optparse from pathlib import Path import pytest import pretend from pip_check_reqs import find_missing_reqs, common, __version__ @pytest.fixture def fake_opts(): class FakeOptParse: class options: requirements_filename = '' paths = ['dummy'] verbose = False debug = False version = False ignore_files = [] ignore_mods = [] options = options() args = ['ham.py'] def __init__(self, usage): pass def add_option(*args, **kw): pass def parse_args(self): return (self.options, self.args) return FakeOptParse def test_find_missing_reqs(monkeypatch, tmp_path: Path): imported_modules = dict(spam=common.FoundModule('spam', 'site-spam/spam.py', [('ham.py', 1)]), shrub=common.FoundModule('shrub', 'site-spam/shrub.py', [('ham.py', 3)]), ignore=common.FoundModule('ignore', 'ignore.py', [('ham.py', 2)])) monkeypatch.setattr(common, 'find_imported_modules', pretend.call_recorder(lambda a: imported_modules)) FakeDist = collections.namedtuple('FakeDist', ['project_name']) installed_distributions = map(FakeDist, ['spam', 'pass']) monkeypatch.setattr(find_missing_reqs, 'get_installed_distributions', pretend.call_recorder(lambda: installed_distributions)) packages_info = [ dict(name='spam', location='site-spam', files=['spam/__init__.py', 'spam/shrub.py']), dict(name='shrub', location='site-spam', files=['shrub.py']), dict(name='pass', location='site-spam', files=['pass.py']), ] monkeypatch.setattr(find_missing_reqs, 'search_packages_info', pretend.call_recorder(lambda x: packages_info)) fake_requirements_file = tmp_path / 'requirements.txt' fake_requirements_file.write_text('spam==1') result = list( find_missing_reqs.find_missing_reqs( options=None, requirements_filename=str(fake_requirements_file), ) ) assert result == [('shrub', [imported_modules['shrub']])] def test_main_failure(monkeypatch, caplog, fake_opts): monkeypatch.setattr(optparse, 'OptionParser', fake_opts) caplog.set_level(logging.WARN) def fake_find_missing_reqs(options, requirements_filename): return [ ( 'missing', [ common.FoundModule( 'missing', 'missing.py', [('location.py', 1)], ) ] ) ] monkeypatch.setattr( find_missing_reqs, 'find_missing_reqs', fake_find_missing_reqs, ) with pytest.raises(SystemExit) as excinfo: find_missing_reqs.main() assert excinfo.value.code == 1 assert caplog.records[0].message == \ 'Missing requirements:' assert caplog.records[1].message == \ 'location.py:1 dist=missing module=missing' def test_main_no_spec(monkeypatch, caplog, fake_opts): fake_opts.args = [] monkeypatch.setattr(optparse, 'OptionParser', fake_opts) monkeypatch.setattr(fake_opts, 'error', pretend.call_recorder(lambda s, e: None), raising=False) with pytest.raises(SystemExit) as excinfo: find_missing_reqs.main() assert excinfo.value.code == 2 assert fake_opts.error.calls @pytest.mark.parametrize(["verbose_cfg", "debug_cfg", "result"], [ (False, False, ['warn']), (True, False, ['info', 'warn']), (False, True, ['debug', 'info', 'warn']), (True, True, ['debug', 'info', 'warn']), ]) def test_logging_config(monkeypatch, caplog, verbose_cfg, debug_cfg, result): class options: requirements_filename = '', paths = ['dummy'] verbose = verbose_cfg debug = debug_cfg version = False ignore_files = [] ignore_mods = [] options = options() class FakeOptParse: def __init__(self, usage): pass def add_option(*args, **kw): pass def parse_args(self): return (options, ['ham.py']) monkeypatch.setattr(optparse, 'OptionParser', FakeOptParse) monkeypatch.setattr( find_missing_reqs, 'find_missing_reqs', lambda options, requirements_filename: [], ) find_missing_reqs.main() for event in [(logging.DEBUG, 'debug'), (logging.INFO, 'info'), (logging.WARN, 'warn')]: find_missing_reqs.log.log(*event) messages = [r.message for r in caplog.records] # first message is always the usage message if verbose_cfg or debug_cfg: assert messages[1:] == result else: assert messages == result def test_main_version(monkeypatch, caplog, fake_opts): fake_opts.options.version = True monkeypatch.setattr(optparse, 'OptionParser', fake_opts) with pytest.raises(SystemExit) as excinfo: find_missing_reqs.main() assert str(excinfo.value) == __version__