././@PaxHeader0000000000000000000000000000003200000000000010210 xustar0026 mtime=1694521663.47774 pip_check_reqs-2.5.3/0000755000076500000240000000000014500054477013333 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521650.0 pip_check_reqs-2.5.3/CHANGELOG.rst0000644000076500000240000000674414500054462015361 0ustar00adamstaff Release History --------------- 2.5.2 - Performance improvements. - Add preliminary support for Windows. 2.5.1 - Fix an issue with importing `__main__`. - Fix an issue with importing packages with periods in their names. 2.5.0 - Support Python 3.10. - Remove support for Python 3.8. - Bump `pip` requirement to 23.2. 2.4.4 - Bump `packaging` requirement to >= 20.5. Older versions of `pip-check-reqs` may be broken with the previously-specified version requirements. 2.4.3 - Improves performance on Python 3.11. 2.4.2 - Added support for Python 3.11. - Added `python_requires` to metadata; from now on, releases of `pip-check-reqs` are marked as compatible with Python 3.8.0 and up. - Made `--version` flag show interpretter version and path to the package which pip-check-reqs is running from, similar to information shown by `pip --version`. - `-V` is now an alias of `--version`. 2.3.2 - Fixed support for pip < 21.3 2.3.1 - Fixed `--skip-incompatible` skipping other requirements too. - Support pip >= 21.3 2.3.0 - Support pip >= 21.2.1 2.2.2 - AST parsing failures will now report tracebacks with a proper filename for the parsed frame, instead of ``. 2.2.1 - Python source is now always read using utf-8, even if default encoding for reading files is set otherwise. 2.2.0 - Added `--skip-incompatible` flag to `pip-extra-reqs`, which makes it ignore requirements with environment markers that are incompatible with the current environment. - Added `--requirements-file` flag to `pip-extra-reqs` and `pip-missing-reqs` commands. This flag makes it possible to specify a path to the requirements file. Previously, `"requirements.txt"` was always used. - Fixed some of the logs not being visible with `-d` and `-v` flags. 2.1.1 - Bug fix: Though Python 2 support was removed from the source code, the published wheel was still universal. The published wheel now explicitly does not support Python 2. Please use version 2.0.4 for Python 2. 2.1.0 - Remove support for Python 2. Please use an older version of this tool if you require that support. - Remove requirement for setuptools. - Support newer versions of pip, including the current version, for more features (20.1.1). Thanks to @Czaki for important parts of this change. 2.0.1 - handled removal of normalize_name from pip.utils - handle packages with no files 2.0 **renamed package to pip_check_reqs** - added tool pip-extra-reqs to find packages installed but not used (contributed by Josh Hesketh) 1.2.1 - relax requirement to 6.0+ 1.2.0 - bumped pip requirement to 6.0.8+ - updated use of pip internals to match that version 1.1.9 - test fixes and cleanup - remove hard-coded simplejson debugging behaviour 1.1.8 - use os.path.realpath to avoid symlink craziness on debian/ubuntu 1.1.7 - tweak to debug output 1.1.6 - add debug (very verbose) run output 1.1.5 - add header to output to make it clearer when in a larger test run - fix tests and self-test 1.1.4 - add --version - remove debug print from released code lol 1.1.3 - fix program to generate exit code useful for testing 1.1.2 - corrected version of vendored search_packages_info() from pip - handle relative imports 1.1.1 - fixed handling of import from __future__ - self-tested and added own requirements.txt - cleaned up usage to require a file or directory to scan (rather than defaulting to ".") - vendored code from pip 1.6dev which fixes bug in search_packages_info until pip 1.6 is released 1.1.0 - implemented --ignore-module ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694392639.0 pip_check_reqs-2.5.3/CONTRIBUTING.rst0000644000076500000240000000106114477460477016010 0ustar00adamstaffContributing ============ Release process --------------- * Update the CHANGELOG on the master branch * Update ``__version__`` in ``pip_check_reqs/__init__.py`` on the master branch. Run the following steps, entering a PyPI API token when prompted: .. code:: sh git checkout master && \ git pull && \ pip install twine && \ pip install -r requirements.txt && \ rm -rf build dist && \ git status # There should be no uncommitted changes. && \ python setup.py sdist bdist_wheel && \ twine upload --username=__token__ -r pypi dist/* ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694212495.0 pip_check_reqs-2.5.3/LICENSE0000644000076500000240000000207014476720617014347 0ustar00adamstaffThe MIT License (MIT) Copyright (c) 2015 Richard Jones Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694212495.0 pip_check_reqs-2.5.3/MANIFEST.in0000644000076500000240000000024714476720617015104 0ustar00adamstaffrecursive-include pip_check_reqs include *.rst include LICENSE include setup.* include tox.ini include requirements.txt include test-requirements.txt include MANIFEST ././@PaxHeader0000000000000000000000000000003200000000000010210 xustar0026 mtime=1694521663.47781 pip_check_reqs-2.5.3/PKG-INFO0000644000076500000240000002012214500054477014425 0ustar00adamstaffMetadata-Version: 2.1 Name: pip_check_reqs Version: 2.5.3 Summary: Find packages that should or should not be in requirements for a project Home-page: https://github.com/r1chardj0n3s/pip-check-reqs Author: Richard Jones Author-email: r1chardj0n3s@gmail.com Maintainer: Adam Dangoor Maintainer-email: adamdangoor@gmail.com License: MIT Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Build Tools Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Requires-Python: >=3.9.0 Provides-Extra: dev License-File: LICENSE |Build Status| |codecov| |PyPI| .. |Build Status| image:: https://github.com/r1chardj0n3s/pip-check-reqs/workflows/CI/badge.svg :target: https://github.com/r1chardj0n3s/pip-check-reqs/actions .. |codecov| image:: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs/branch/master/graph/badge.svg :target: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs .. |PyPI| image:: https://badge.fury.io/py/pip-check-reqs.svg :target: https://badge.fury.io/py/pip-check-reqs pip-check-reqs ============== It happens: you start using a module in your project and it works and you don't realise that it's only being included in your `virtualenv`_ because it's a dependency of a package you're using. pip-missing-reqs finds those modules so you can include them in the `requirements.txt`_ for the project. Alternatively, you have a long-running project that has some packages in requirements.txt that are no longer actively used in the codebase. The pip-extra-reqs tool will find those modules so you can remove them. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ .. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files Assuming your project follows a layout like the suggested sample project:: setup.py setup.cfg requirements.txt sample/__init__.py sample/sample.py sample/tests/test_sample.py Basic usage, running in your project directory:: pip-missing-reqs --ignore-file=sample/tests/* sample This will find all imports in the code in "sample" and check that the packages those modules belong to are in the requirements.txt file. Additionally it is possible to check that there are no dependencies in requirements.txt that are then unused in the project:: pip-extra-reqs --ignore-file=sample/tests/* sample This would find anything that is listed in requirements.txt but that is not imported by sample. Sample tox.ini configuration ---------------------------- To make your life easier, copy something like this into your tox.ini:: [testenv:pip-check-reqs] deps=-rrequirements.txt commands= pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample Excluding test files (or others) from this check ------------------------------------------------ Your test files will sometimes be present in the same directory as your application source ("sample" in the above examples). The requirements for those tests generally should not be in the requirements.txt file, and you don't want this tool to generate false hits for those. You may exclude those test files from your check using the `--ignore-file` option (shorthand is `-f`). Multiple instances of the option are allowed. Excluding modules from the check -------------------------------- If your project has modules which are conditionally imported, or requirements which are conditionally included, you may exclude certain modules from the check by name (or glob pattern) using `--ignore-module` (shorthand is `-m`):: # ignore the module spam pip-missing-reqs --ignore-module=spam sample # ignore the whole package spam as well pip-missing-reqs --ignore-module=spam --ignore-module=spam.* sample Using pyproject.toml instead of requirements.txt ------------------------------------------------ If your project uses `pyproject.toml` instead of `requirements.txt`, you can use an external tool like `pdm` to convert it to `requirements.txt`:: # requires `pip install pdm` pdm export --pyproject > requirements.txt Then you can use `pip-missing-reqs` and `pip-extra-reqs` as usual. With Thanks To -------------- Josh Hesketh -- who refactored code and contributed the pip-extra-reqs tool. Wil Cooley -- who handled the removal of normalize_name and fixed some bugs. Release History --------------- 2.5.2 - Performance improvements. - Add preliminary support for Windows. 2.5.1 - Fix an issue with importing `__main__`. - Fix an issue with importing packages with periods in their names. 2.5.0 - Support Python 3.10. - Remove support for Python 3.8. - Bump `pip` requirement to 23.2. 2.4.4 - Bump `packaging` requirement to >= 20.5. Older versions of `pip-check-reqs` may be broken with the previously-specified version requirements. 2.4.3 - Improves performance on Python 3.11. 2.4.2 - Added support for Python 3.11. - Added `python_requires` to metadata; from now on, releases of `pip-check-reqs` are marked as compatible with Python 3.8.0 and up. - Made `--version` flag show interpretter version and path to the package which pip-check-reqs is running from, similar to information shown by `pip --version`. - `-V` is now an alias of `--version`. 2.3.2 - Fixed support for pip < 21.3 2.3.1 - Fixed `--skip-incompatible` skipping other requirements too. - Support pip >= 21.3 2.3.0 - Support pip >= 21.2.1 2.2.2 - AST parsing failures will now report tracebacks with a proper filename for the parsed frame, instead of ``. 2.2.1 - Python source is now always read using utf-8, even if default encoding for reading files is set otherwise. 2.2.0 - Added `--skip-incompatible` flag to `pip-extra-reqs`, which makes it ignore requirements with environment markers that are incompatible with the current environment. - Added `--requirements-file` flag to `pip-extra-reqs` and `pip-missing-reqs` commands. This flag makes it possible to specify a path to the requirements file. Previously, `"requirements.txt"` was always used. - Fixed some of the logs not being visible with `-d` and `-v` flags. 2.1.1 - Bug fix: Though Python 2 support was removed from the source code, the published wheel was still universal. The published wheel now explicitly does not support Python 2. Please use version 2.0.4 for Python 2. 2.1.0 - Remove support for Python 2. Please use an older version of this tool if you require that support. - Remove requirement for setuptools. - Support newer versions of pip, including the current version, for more features (20.1.1). Thanks to @Czaki for important parts of this change. 2.0.1 - handled removal of normalize_name from pip.utils - handle packages with no files 2.0 **renamed package to pip_check_reqs** - added tool pip-extra-reqs to find packages installed but not used (contributed by Josh Hesketh) 1.2.1 - relax requirement to 6.0+ 1.2.0 - bumped pip requirement to 6.0.8+ - updated use of pip internals to match that version 1.1.9 - test fixes and cleanup - remove hard-coded simplejson debugging behaviour 1.1.8 - use os.path.realpath to avoid symlink craziness on debian/ubuntu 1.1.7 - tweak to debug output 1.1.6 - add debug (very verbose) run output 1.1.5 - add header to output to make it clearer when in a larger test run - fix tests and self-test 1.1.4 - add --version - remove debug print from released code lol 1.1.3 - fix program to generate exit code useful for testing 1.1.2 - corrected version of vendored search_packages_info() from pip - handle relative imports 1.1.1 - fixed handling of import from __future__ - self-tested and added own requirements.txt - cleaned up usage to require a file or directory to scan (rather than defaulting to ".") - vendored code from pip 1.6dev which fixes bug in search_packages_info until pip 1.6 is released 1.1.0 - implemented --ignore-module ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694392639.0 pip_check_reqs-2.5.3/README.rst0000644000076500000240000000747614477460477015056 0ustar00adamstaff|Build Status| |codecov| |PyPI| .. |Build Status| image:: https://github.com/r1chardj0n3s/pip-check-reqs/workflows/CI/badge.svg :target: https://github.com/r1chardj0n3s/pip-check-reqs/actions .. |codecov| image:: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs/branch/master/graph/badge.svg :target: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs .. |PyPI| image:: https://badge.fury.io/py/pip-check-reqs.svg :target: https://badge.fury.io/py/pip-check-reqs pip-check-reqs ============== It happens: you start using a module in your project and it works and you don't realise that it's only being included in your `virtualenv`_ because it's a dependency of a package you're using. pip-missing-reqs finds those modules so you can include them in the `requirements.txt`_ for the project. Alternatively, you have a long-running project that has some packages in requirements.txt that are no longer actively used in the codebase. The pip-extra-reqs tool will find those modules so you can remove them. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ .. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files Assuming your project follows a layout like the suggested sample project:: setup.py setup.cfg requirements.txt sample/__init__.py sample/sample.py sample/tests/test_sample.py Basic usage, running in your project directory:: pip-missing-reqs --ignore-file=sample/tests/* sample This will find all imports in the code in "sample" and check that the packages those modules belong to are in the requirements.txt file. Additionally it is possible to check that there are no dependencies in requirements.txt that are then unused in the project:: pip-extra-reqs --ignore-file=sample/tests/* sample This would find anything that is listed in requirements.txt but that is not imported by sample. Sample tox.ini configuration ---------------------------- To make your life easier, copy something like this into your tox.ini:: [testenv:pip-check-reqs] deps=-rrequirements.txt commands= pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample Excluding test files (or others) from this check ------------------------------------------------ Your test files will sometimes be present in the same directory as your application source ("sample" in the above examples). The requirements for those tests generally should not be in the requirements.txt file, and you don't want this tool to generate false hits for those. You may exclude those test files from your check using the `--ignore-file` option (shorthand is `-f`). Multiple instances of the option are allowed. Excluding modules from the check -------------------------------- If your project has modules which are conditionally imported, or requirements which are conditionally included, you may exclude certain modules from the check by name (or glob pattern) using `--ignore-module` (shorthand is `-m`):: # ignore the module spam pip-missing-reqs --ignore-module=spam sample # ignore the whole package spam as well pip-missing-reqs --ignore-module=spam --ignore-module=spam.* sample Using pyproject.toml instead of requirements.txt ------------------------------------------------ If your project uses `pyproject.toml` instead of `requirements.txt`, you can use an external tool like `pdm` to convert it to `requirements.txt`:: # requires `pip install pdm` pdm export --pyproject > requirements.txt Then you can use `pip-missing-reqs` and `pip-extra-reqs` as usual. With Thanks To -------------- Josh Hesketh -- who refactored code and contributed the pip-extra-reqs tool. Wil Cooley -- who handled the removal of normalize_name and fixed some bugs. ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1694521663.475945 pip_check_reqs-2.5.3/pip_check_reqs/0000755000076500000240000000000014500054477016312 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521642.0 pip_check_reqs-2.5.3/pip_check_reqs/__init__.py0000644000076500000240000000012114500054452020406 0ustar00adamstaff"""Package for finding missing and extra requirements.""" __version__ = "2.5.3" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694520162.0 pip_check_reqs-2.5.3/pip_check_reqs/common.py0000644000076500000240000002026514500051542020147 0ustar00adamstaff"""Common functions.""" from __future__ import annotations import ast import fnmatch import logging import os import sys from dataclasses import dataclass, field from importlib.util import find_spec from pathlib import Path from typing import ( Callable, Generator, Iterable, ) from packaging.markers import Marker from packaging.utils import NormalizedName, canonicalize_name from pip._internal.network.session import PipSession from pip._internal.req.constructors import install_req_from_line from pip._internal.req.req_file import ParsedRequirement, parse_requirements from . import __version__ log = logging.getLogger(__name__) @dataclass class FoundModule: """A module with uses in the source.""" modname: str filename: Path locations: list[tuple[str, int]] = field(default_factory=list) def __post_init__(self) -> None: self.filename = Path(self.filename).resolve() class _ImportVisitor(ast.NodeVisitor): def __init__(self, ignore_modules_function: Callable[[str], bool]) -> None: super().__init__() self._ignore_modules_function = ignore_modules_function self._modules: dict[str, FoundModule] = {} self._location: str | None = None def set_location(self, *, location: str) -> None: self._location = location # Ignore the name error as we are overriding the method. def visit_Import( # noqa: N802, pylint: disable=invalid-name self, node: ast.Import, ) -> None: for alias in node.names: self._add_module(alias.name, node.lineno) # Ignore the name error as we are overriding the method. def visit_ImportFrom( # noqa: N802, pylint: disable=invalid-name self, node: ast.ImportFrom, ) -> None: if node.module == "__future__": # not an actual module return for alias in node.names: if node.module is None or node.level != 0: # relative import continue self._add_module(node.module + "." + alias.name, node.lineno) def _add_module(self, modname: str, lineno: int) -> None: if self._ignore_modules_function(modname): return modname_parts_progress: list[str] = [] for modname_part in modname.split("."): name = ".".join([*modname_parts_progress, modname_part]) try: module_spec = find_spec(name=name) except ValueError: # The module has no __spec__ attribute. # For example, if importing __main__. return if module_spec is None: # The component specified at this point is not installed. return if module_spec.origin is None: modname_parts_progress.append(modname_part) continue modpath = module_spec.origin if modpath == "frozen": # Frozen modules are modules written in Python whose compiled # byte-code object is incorporated into a custom-built Python # interpreter by Python's freeze utility. continue modpath_path = Path(modpath) modname = module_spec.name if modname not in self._modules: if modpath_path.is_file(): if modpath_path.name == "__init__.py": modpath_path = modpath_path.parent else: # We have this empty "else" so that we are # not tempted to combine the "is file" and "is # __init__" checks, and to make sure we have coverage # for this case. pass self._modules[modname] = FoundModule( modname=modname, filename=modpath_path, ) assert isinstance(self._location, str) self._modules[modname].locations.append((self._location, lineno)) return def finalise(self) -> dict[str, FoundModule]: return self._modules def pyfiles(root: Path) -> Generator[Path, None, None]: if root.is_file(): if root.suffix == ".py": yield root.absolute() else: msg = f"{root} is not a python file or directory" raise ValueError(msg) else: for item in root.rglob("*.py"): yield item.absolute() def find_imported_modules( *, paths: Iterable[Path], ignore_files_function: Callable[[str], bool], ignore_modules_function: Callable[[str], bool], ) -> dict[str, FoundModule]: vis = _ImportVisitor(ignore_modules_function=ignore_modules_function) for path in paths: for filename in pyfiles(path): if ignore_files_function(str(filename)): log.info("ignoring: %s", filename) continue log.debug("scanning: %s", filename) content = filename.read_text(encoding="utf-8") vis.set_location(location=str(filename)) vis.visit(ast.parse(content, str(filename))) return vis.finalise() def find_required_modules( *, ignore_requirements_function: Callable[ [str | ParsedRequirement], bool, ], skip_incompatible: bool, requirements_filename: Path, ) -> set[NormalizedName]: explicit: set[NormalizedName] = set() for requirement in parse_requirements( str(requirements_filename), session=PipSession(), ): requirement_name = install_req_from_line( requirement.requirement, ).name assert isinstance(requirement_name, str) if ignore_requirements_function(requirement): log.debug("ignoring requirement: %s", requirement_name) continue if skip_incompatible: requirement_string = requirement.requirement if not has_compatible_markers(full_requirement=requirement_string): log.debug( "ignoring requirement (incompatible environment " "marker): %s", requirement_string, ) continue log.debug("found requirement: %s", requirement_name) explicit.add(canonicalize_name(requirement_name)) return explicit def has_compatible_markers(*, full_requirement: str) -> bool: if ";" not in full_requirement: return True # No environment marker. enviroment_marker = full_requirement.split(";")[1] if not enviroment_marker: return True # Empty environment marker. return Marker(enviroment_marker).evaluate() def package_path(*, path: Path) -> Path | None: """Return the package path for a given Python package sentinel file. Return None if the path is not a sentinel file. A sentinel file is the __init__.py or its compiled variants. """ if path.parent == path.parent.parent: return None if path.name not in ("__init__.py", "__init__.pyc", "__init__.pyo"): return None return path.parent def ignorer(*, ignore_cfg: list[str]) -> Callable[..., bool]: if not ignore_cfg: return lambda _: False def ignorer_function( candidate: str | ParsedRequirement, ignore_cfg: list[str] = ignore_cfg, ) -> bool: for ignore in ignore_cfg: if isinstance(candidate, str): candidate_path = candidate else: optional_candidate_path = install_req_from_line( candidate.requirement, ).name assert isinstance(optional_candidate_path, str) candidate_path = optional_candidate_path if fnmatch.fnmatch(candidate_path, ignore): return True if fnmatch.fnmatch(os.path.relpath(candidate_path), ignore): return True return False return ignorer_function def version_info() -> str: major, minor, patch = sys.version_info[:3] python_version = f"{major}.{minor}.{patch}" parent_directory = Path(__file__).parent.resolve() return ( f"pip-check-reqs {__version__} " f"from {parent_directory} " f"(python {python_version})" ) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694515069.0 pip_check_reqs-2.5.3/pip_check_reqs/find_extra_reqs.py0000644000076500000240000001707414500037575022052 0ustar00adamstaff"""Find extra requirements.""" from __future__ import annotations import argparse import collections import importlib.metadata import logging import os import sys from functools import cache from pathlib import Path from typing import TYPE_CHECKING, Callable, Iterable from unittest import mock from packaging.utils import NormalizedName, canonicalize_name from pip._internal.commands.show import ( _PackageInfo, # pyright: ignore[reportPrivateUsage] search_packages_info, ) from pip_check_reqs import common from pip_check_reqs.common import version_info if TYPE_CHECKING: from pip._internal.req.req_file import ParsedRequirement log = logging.getLogger(__name__) # This is a slow operation. # It only happens once when calling the CLI, but it is hit many times in # tests. # We cache the result to speed up tests. @cache def get_packages_info() -> list[_PackageInfo]: all_pkgs = [ dist.metadata["Name"] for dist in importlib.metadata.distributions() ] # On Python 3.11 (and maybe higher), setting this environment variable # dramatically improves speeds. # See https://github.com/r1chardj0n3s/pip-check-reqs/issues/123. with mock.patch.dict(os.environ, {"_PIP_USE_IMPORTLIB_METADATA": "False"}): return list(search_packages_info(query=all_pkgs)) def find_extra_reqs( *, requirements_filename: Path, paths: Iterable[Path], ignore_files_function: Callable[[str], bool], ignore_modules_function: Callable[[str], bool], ignore_requirements_function: Callable[ [str | ParsedRequirement], bool, ], skip_incompatible: bool, ) -> list[str]: # 1. find files used by imports in the code (as best we can without # executing) used_modules = common.find_imported_modules( paths=paths, ignore_files_function=ignore_files_function, ignore_modules_function=ignore_modules_function, ) installed_files: dict[Path, str] = {} packages_info = get_packages_info() here = Path().resolve() for package in packages_info: package_name = package.name package_location = package.location package_files: list[str] = [] for item in package.files or []: item_location_rel = Path(package_location) / item item_location = item_location_rel.resolve() try: relative_item_location = item_location.relative_to(here) except ValueError: # Ideally we would use Pathlib.is_relative_to rather than # checking for a ValueError, but that is only available in # Python 3.9+. relative_item_location = item_location package_files.append(str(relative_item_location)) log.debug( "installed package: %s (at %s)", package_name, package_location, ) for package_file in package_files: path = Path(package_location) / package_file path = path.resolve() installed_files[path] = package_name package_path = common.package_path(path=path) if package_path: # we've seen a package file so add the bare package directory # to the installed list as well as we might want to look up # a package by its directory path later installed_files[package_path] = package_name # 3. match imported modules against those packages used: collections.defaultdict[ NormalizedName, list[common.FoundModule], ] = collections.defaultdict(list) for modname, info in used_modules.items(): # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(installed_files[info.filename]) log.debug( "used module: %s (from package %s)", modname, installed_files[info.filename], ) used[used_name].append(info) else: log.debug( "used module: %s (from file %s, assuming stdlib or local)", modname, info.filename, ) # 4. compare with requirements explicit = common.find_required_modules( ignore_requirements_function=ignore_requirements_function, skip_incompatible=skip_incompatible, requirements_filename=requirements_filename, ) return [name for name in explicit if name not in used] def main(arguments: list[str] | None = None) -> None: """pip-extra-reqs entry point.""" usage = "usage: %prog [options] files or directories" parser = argparse.ArgumentParser(usage) parser.add_argument("paths", type=Path, nargs="*") parser.add_argument( "--requirements-file", dest="requirements_filename", type=Path, metavar="PATH", default=Path("requirements.txt"), help="path to the requirements file " '(defaults to "requirements.txt")', ) parser.add_argument( "-f", "--ignore-file", dest="ignore_files", action="append", default=[], help="file paths globs to ignore", ) parser.add_argument( "-m", "--ignore-module", dest="ignore_mods", action="append", default=[], help="used module names (globs are ok) to ignore", ) parser.add_argument( "-r", "--ignore-requirement", dest="ignore_reqs", action="append", default=[], help="reqs in requirements to ignore", ) parser.add_argument( "-s", "--skip-incompatible", dest="skip_incompatible", action="store_true", default=False, help="skip requirements that have incompatible environment markers", ) parser.add_argument( "-v", "--verbose", dest="verbose", action="store_true", default=False, help="be more verbose", ) parser.add_argument( "-d", "--debug", dest="debug", action="store_true", default=False, help="be *really* verbose", ) parser.add_argument( "-V", "--version", dest="version", action="store_true", default=False, help="display version information", ) parse_result = parser.parse_args(arguments) if parse_result.version: sys.stdout.write(version_info() + "\n") sys.exit(0) if not parse_result.paths: parser.error("no source files or directories specified") ignore_files = common.ignorer(ignore_cfg=parse_result.ignore_files) ignore_mods = common.ignorer(ignore_cfg=parse_result.ignore_mods) ignore_reqs = common.ignorer(ignore_cfg=parse_result.ignore_reqs) logging.basicConfig(format="%(message)s") if parse_result.debug: level = logging.DEBUG elif parse_result.verbose: level = logging.INFO else: level = logging.WARN log.setLevel(level) common.log.setLevel(level) log.info(version_info()) extras = find_extra_reqs( requirements_filename=parse_result.requirements_filename, paths=parse_result.paths, ignore_files_function=ignore_files, ignore_modules_function=ignore_mods, ignore_requirements_function=ignore_reqs, skip_incompatible=parse_result.skip_incompatible, ) if extras: log.warning("Extra requirements:") for name in extras: message = f"{name} in {parse_result.requirements_filename}" log.warning(message) if extras: sys.exit(1) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694515069.0 pip_check_reqs-2.5.3/pip_check_reqs/find_missing_reqs.py0000644000076500000240000001653414500037575022400 0ustar00adamstaff"""Find missing requirements.""" from __future__ import annotations import argparse import collections import importlib.metadata import logging import os import sys from functools import cache from pathlib import Path from typing import Callable, Iterable from unittest import mock from packaging.utils import NormalizedName, canonicalize_name from pip._internal.commands.show import ( _PackageInfo, # pyright: ignore[reportPrivateUsage] search_packages_info, ) from pip._internal.network.session import PipSession from pip._internal.req.constructors import install_req_from_line from pip._internal.req.req_file import parse_requirements from pip_check_reqs import common from pip_check_reqs.common import FoundModule, version_info log = logging.getLogger(__name__) # This is a slow operation. # It only happens once when calling the CLI, but it is hit many times in # tests. # We cache the result to speed up tests. @cache def get_packages_info() -> list[_PackageInfo]: all_pkgs = [ dist.metadata["Name"] for dist in importlib.metadata.distributions() ] # On Python 3.11 (and maybe higher), setting this environment variable # dramatically improves speeds. # See https://github.com/r1chardj0n3s/pip-check-reqs/issues/123. with mock.patch.dict(os.environ, {"_PIP_USE_IMPORTLIB_METADATA": "False"}): return list(search_packages_info(query=all_pkgs)) def find_missing_reqs( requirements_filename: Path, paths: Iterable[Path], ignore_files_function: Callable[[str], bool], ignore_modules_function: Callable[[str], bool], ) -> list[tuple[NormalizedName, list[FoundModule]]]: # 1. find files used by imports in the code (as best we can without # executing) used_modules = common.find_imported_modules( paths=paths, ignore_files_function=ignore_files_function, ignore_modules_function=ignore_modules_function, ) installed_files: dict[Path, str] = {} packages_info = get_packages_info() here = Path().resolve() for package in packages_info: package_name = package.name package_location = package.location package_files: list[str] = [] for item in package.files or []: item_location_rel = Path(package_location) / item item_location = item_location_rel.resolve() try: relative_item_location = item_location.relative_to(here) except ValueError: # Ideally we would use Pathlib.is_relative_to rather than # checking for a ValueError, but that is only available in # Python 3.9+. relative_item_location = item_location package_files.append(str(relative_item_location)) log.debug( "installed package: %s (at %s)", package_name, package_location, ) for package_file in package_files: path = Path(package_location) / package_file path = path.resolve() installed_files[path] = package_name package_path = common.package_path(path=path) if package_path: # we've seen a package file so add the bare package directory # to the installed list as well as we might want to look up # a package by its directory path later installed_files[package_path] = package_name # 3. match imported modules against those packages used: collections.defaultdict[ NormalizedName, list[common.FoundModule], ] = collections.defaultdict(list) for modname, info in used_modules.items(): # probably standard library if it's not in the files list if info.filename in installed_files: used_name = canonicalize_name(name=installed_files[info.filename]) log.debug( "used module: %s (from package %s)", modname, installed_files[info.filename], ) used[used_name].append(info) else: log.debug( "used module: %s (from file %s, assuming stdlib or local)", modname, info.filename, ) # 4. compare with requirements explicit: set[NormalizedName] = set() for requirement in parse_requirements( str(requirements_filename), session=PipSession(), ): requirement_name = install_req_from_line( requirement.requirement, ).name assert isinstance(requirement_name, str) log.debug("found requirement: %s", requirement_name) explicit.add(canonicalize_name(requirement_name)) return [(name, used[name]) for name in used if name not in explicit] def main(arguments: list[str] | None = None) -> None: usage = "usage: %prog [options] files or directories" parser = argparse.ArgumentParser(usage) parser.add_argument("paths", type=Path, nargs="*") parser.add_argument( "--requirements-file", dest="requirements_filename", metavar="PATH", type=Path, default="requirements.txt", help="path to the requirements file " '(defaults to "requirements.txt")', ) parser.add_argument( "-f", "--ignore-file", dest="ignore_files", action="append", default=[], help="file paths globs to ignore", ) parser.add_argument( "-m", "--ignore-module", dest="ignore_mods", action="append", default=[], help="used module names (globs are ok) to ignore", ) parser.add_argument( "-v", "--verbose", dest="verbose", action="store_true", default=False, help="be more verbose", ) parser.add_argument( "-d", "--debug", dest="debug", action="store_true", default=False, help="be *really* verbose", ) parser.add_argument( "-V", "--version", dest="version", action="store_true", default=False, help="display version information", ) parse_result = parser.parse_args(arguments) if parse_result.version: sys.stdout.write(version_info() + "\n") sys.exit(0) if not parse_result.paths: parser.error("no source files or directories specified") ignore_files = common.ignorer(ignore_cfg=parse_result.ignore_files) ignore_mods = common.ignorer(ignore_cfg=parse_result.ignore_mods) logging.basicConfig(format="%(message)s") if parse_result.debug: level = logging.DEBUG elif parse_result.verbose: level = logging.INFO else: level = logging.WARN log.setLevel(level) common.log.setLevel(level) log.info(version_info()) missing = find_missing_reqs( requirements_filename=parse_result.requirements_filename, paths=parse_result.paths, ignore_files_function=ignore_files, ignore_modules_function=ignore_mods, ) if missing: log.warning("Missing requirements:") for name, uses in missing: for use in uses: for filename, lineno in use.locations: log.warning( "%s:%s dist=%s module=%s", filename, lineno, name, use.modname, ) if missing: sys.exit(1) ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1694521663.4767878 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/0000755000076500000240000000000014500054477020004 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521663.0 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/PKG-INFO0000644000076500000240000002012214500054477021076 0ustar00adamstaffMetadata-Version: 2.1 Name: pip-check-reqs Version: 2.5.3 Summary: Find packages that should or should not be in requirements for a project Home-page: https://github.com/r1chardj0n3s/pip-check-reqs Author: Richard Jones Author-email: r1chardj0n3s@gmail.com Maintainer: Adam Dangoor Maintainer-email: adamdangoor@gmail.com License: MIT Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Topic :: Software Development :: Build Tools Classifier: License :: OSI Approved :: MIT License Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Requires-Python: >=3.9.0 Provides-Extra: dev License-File: LICENSE |Build Status| |codecov| |PyPI| .. |Build Status| image:: https://github.com/r1chardj0n3s/pip-check-reqs/workflows/CI/badge.svg :target: https://github.com/r1chardj0n3s/pip-check-reqs/actions .. |codecov| image:: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs/branch/master/graph/badge.svg :target: https://codecov.io/gh/r1chardj0n3s/pip-check-reqs .. |PyPI| image:: https://badge.fury.io/py/pip-check-reqs.svg :target: https://badge.fury.io/py/pip-check-reqs pip-check-reqs ============== It happens: you start using a module in your project and it works and you don't realise that it's only being included in your `virtualenv`_ because it's a dependency of a package you're using. pip-missing-reqs finds those modules so you can include them in the `requirements.txt`_ for the project. Alternatively, you have a long-running project that has some packages in requirements.txt that are no longer actively used in the codebase. The pip-extra-reqs tool will find those modules so you can remove them. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ .. _`requirements.txt`: https://pip.pypa.io/en/latest/user_guide.html#requirements-files Assuming your project follows a layout like the suggested sample project:: setup.py setup.cfg requirements.txt sample/__init__.py sample/sample.py sample/tests/test_sample.py Basic usage, running in your project directory:: pip-missing-reqs --ignore-file=sample/tests/* sample This will find all imports in the code in "sample" and check that the packages those modules belong to are in the requirements.txt file. Additionally it is possible to check that there are no dependencies in requirements.txt that are then unused in the project:: pip-extra-reqs --ignore-file=sample/tests/* sample This would find anything that is listed in requirements.txt but that is not imported by sample. Sample tox.ini configuration ---------------------------- To make your life easier, copy something like this into your tox.ini:: [testenv:pip-check-reqs] deps=-rrequirements.txt commands= pip-missing-reqs --ignore-file=sample/tests/* sample pip-extra-reqs --ignore-file=sample/tests/* sample Excluding test files (or others) from this check ------------------------------------------------ Your test files will sometimes be present in the same directory as your application source ("sample" in the above examples). The requirements for those tests generally should not be in the requirements.txt file, and you don't want this tool to generate false hits for those. You may exclude those test files from your check using the `--ignore-file` option (shorthand is `-f`). Multiple instances of the option are allowed. Excluding modules from the check -------------------------------- If your project has modules which are conditionally imported, or requirements which are conditionally included, you may exclude certain modules from the check by name (or glob pattern) using `--ignore-module` (shorthand is `-m`):: # ignore the module spam pip-missing-reqs --ignore-module=spam sample # ignore the whole package spam as well pip-missing-reqs --ignore-module=spam --ignore-module=spam.* sample Using pyproject.toml instead of requirements.txt ------------------------------------------------ If your project uses `pyproject.toml` instead of `requirements.txt`, you can use an external tool like `pdm` to convert it to `requirements.txt`:: # requires `pip install pdm` pdm export --pyproject > requirements.txt Then you can use `pip-missing-reqs` and `pip-extra-reqs` as usual. With Thanks To -------------- Josh Hesketh -- who refactored code and contributed the pip-extra-reqs tool. Wil Cooley -- who handled the removal of normalize_name and fixed some bugs. Release History --------------- 2.5.2 - Performance improvements. - Add preliminary support for Windows. 2.5.1 - Fix an issue with importing `__main__`. - Fix an issue with importing packages with periods in their names. 2.5.0 - Support Python 3.10. - Remove support for Python 3.8. - Bump `pip` requirement to 23.2. 2.4.4 - Bump `packaging` requirement to >= 20.5. Older versions of `pip-check-reqs` may be broken with the previously-specified version requirements. 2.4.3 - Improves performance on Python 3.11. 2.4.2 - Added support for Python 3.11. - Added `python_requires` to metadata; from now on, releases of `pip-check-reqs` are marked as compatible with Python 3.8.0 and up. - Made `--version` flag show interpretter version and path to the package which pip-check-reqs is running from, similar to information shown by `pip --version`. - `-V` is now an alias of `--version`. 2.3.2 - Fixed support for pip < 21.3 2.3.1 - Fixed `--skip-incompatible` skipping other requirements too. - Support pip >= 21.3 2.3.0 - Support pip >= 21.2.1 2.2.2 - AST parsing failures will now report tracebacks with a proper filename for the parsed frame, instead of ``. 2.2.1 - Python source is now always read using utf-8, even if default encoding for reading files is set otherwise. 2.2.0 - Added `--skip-incompatible` flag to `pip-extra-reqs`, which makes it ignore requirements with environment markers that are incompatible with the current environment. - Added `--requirements-file` flag to `pip-extra-reqs` and `pip-missing-reqs` commands. This flag makes it possible to specify a path to the requirements file. Previously, `"requirements.txt"` was always used. - Fixed some of the logs not being visible with `-d` and `-v` flags. 2.1.1 - Bug fix: Though Python 2 support was removed from the source code, the published wheel was still universal. The published wheel now explicitly does not support Python 2. Please use version 2.0.4 for Python 2. 2.1.0 - Remove support for Python 2. Please use an older version of this tool if you require that support. - Remove requirement for setuptools. - Support newer versions of pip, including the current version, for more features (20.1.1). Thanks to @Czaki for important parts of this change. 2.0.1 - handled removal of normalize_name from pip.utils - handle packages with no files 2.0 **renamed package to pip_check_reqs** - added tool pip-extra-reqs to find packages installed but not used (contributed by Josh Hesketh) 1.2.1 - relax requirement to 6.0+ 1.2.0 - bumped pip requirement to 6.0.8+ - updated use of pip internals to match that version 1.1.9 - test fixes and cleanup - remove hard-coded simplejson debugging behaviour 1.1.8 - use os.path.realpath to avoid symlink craziness on debian/ubuntu 1.1.7 - tweak to debug output 1.1.6 - add debug (very verbose) run output 1.1.5 - add header to output to make it clearer when in a larger test run - fix tests and self-test 1.1.4 - add --version - remove debug print from released code lol 1.1.3 - fix program to generate exit code useful for testing 1.1.2 - corrected version of vendored search_packages_info() from pip - handle relative imports 1.1.1 - fixed handling of import from __future__ - self-tested and added own requirements.txt - cleaned up usage to require a file or directory to scan (rather than defaulting to ".") - vendored code from pip 1.6dev which fixes bug in search_packages_info until pip 1.6 is released 1.1.0 - implemented --ignore-module ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521663.0 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/SOURCES.txt0000644000076500000240000000107114500054477021667 0ustar00adamstaffCHANGELOG.rst CONTRIBUTING.rst LICENSE MANIFEST.in README.rst pyproject.toml requirements.txt setup.cfg setup.py test-requirements.txt pip_check_reqs/__init__.py pip_check_reqs/common.py pip_check_reqs/find_extra_reqs.py pip_check_reqs/find_missing_reqs.py pip_check_reqs.egg-info/PKG-INFO pip_check_reqs.egg-info/SOURCES.txt pip_check_reqs.egg-info/dependency_links.txt pip_check_reqs.egg-info/entry_points.txt pip_check_reqs.egg-info/requires.txt pip_check_reqs.egg-info/top_level.txt tests/test_common.py tests/test_find_extra_reqs.py tests/test_find_missing_reqs.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521663.0 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/dependency_links.txt0000644000076500000240000000000114500054477024052 0ustar00adamstaff ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521663.0 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/entry_points.txt0000644000076500000240000000020014500054477023272 0ustar00adamstaff[console_scripts] pip-extra-reqs = pip_check_reqs.find_extra_reqs:main pip-missing-reqs = pip_check_reqs.find_missing_reqs:main ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521663.0 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/requires.txt0000644000076500000240000000017114500054477022403 0ustar00adamstaffpackaging>=20.5 pip>=23.2 [dev] black ruff mypy pyenchant pylint pyright pytest pytest-cov types-setuptools ruamel.yaml ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521663.0 pip_check_reqs-2.5.3/pip_check_reqs.egg-info/top_level.txt0000644000076500000240000000001714500054477022534 0ustar00adamstaffpip_check_reqs ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694515069.0 pip_check_reqs-2.5.3/pyproject.toml0000644000076500000240000001030014500037575016241 0ustar00adamstaff[tool.pylint] [tool.pylint.'MASTER'] # Pickle collected data for later comparisons. persistent = true # Use multiple processes to speed up Pylint. jobs = 0 # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins = [ 'pylint.extensions.docparams', 'pylint.extensions.no_self_use', ] # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension = false [tool.pylint.'MESSAGES CONTROL'] # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. enable = [ 'spelling', 'useless-suppression', ] # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable = [ 'too-few-public-methods', 'too-many-locals', 'too-many-arguments', 'too-many-instance-attributes', 'too-many-return-statements', 'too-many-lines', 'locally-disabled', # Let ruff handle long lines 'line-too-long', # Let ruff handle unused imports 'unused-import', # Let isort deal with sorting 'ungrouped-imports', # We don't need everything to be documented because of mypy 'missing-type-doc', 'missing-return-type-doc', # Too difficult to please 'duplicate-code', # Let ruff handle imports 'wrong-import-order', # It would be nice to add this, but it's too much work "missing-function-docstring", # We will remove this in issue 97 "deprecated-module", ] [tool.pylint.'FORMAT'] # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt = false [tool.pylint.'SPELLING'] # Spelling dictionary name. Available dictionaries: none. To make it working # install python-enchant package. spelling-dict = 'en_US' # A path to a file that contains private dictionary; one word per line. spelling-private-dict-file = 'spelling_private_dict.txt' # Tells whether to store unknown words to indicated private dictionary in # --spelling-private-dict-file option instead of raising a message. spelling-store-unknown-words = 'no' [tool.black] line-length = 79 [tool.mypy] strict = true [tool.ruff] target-version = "py38" select = ["ALL"] ignore = [ # We do not annotate the type of 'self', or 'cls'. "ANN101", # We are missing too many docstrings to quickly fix now. "D100", "D103", "D104", "D105", "D203", "D213", # Allow functions to take as many arguments as they want. "PLR0913", # Allow 'assert' in tests as it is the standard for pytest. # Also, allow 'assert' in other code as it is the standard for Python type hint # narrowing - see # https://mypy.readthedocs.io/en/stable/type_narrowing.html#type-narrowing-expressions. "S101", ] line-length = 79 # Do not automatically remove commented out code. # We comment out code during development, and with VSCode auto-save, this code # is sometimes annoyingly removed. unfixable = ["ERA001"] [tool.coverage.run] branch = true [tool.coverage.report] exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", ] [tool.pyright] typeCheckingMode = "strict" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694346740.0 pip_check_reqs-2.5.3/requirements.txt0000644000076500000240000000014314477326764016633 0ustar00adamstaffpackaging >= 20.5 # Pinned pip versions are matched in the GitHub workflows CI matrix. pip >= 23.2 ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1694521663.4780042 pip_check_reqs-2.5.3/setup.cfg0000644000076500000240000000004614500054477015154 0ustar00adamstaff[egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694392455.0 pip_check_reqs-2.5.3/setup.py0000644000076500000240000000412514477460207015054 0ustar00adamstafffrom __future__ import annotations from pathlib import Path from setuptools import setup from pip_check_reqs import __version__ here = Path.resolve(Path(__file__).parent) def _get_dependencies(requirements_file: Path) -> list[str]: """Return requirements from a requirements file. This expects a requirements file with no ``--find-links`` lines. """ lines = requirements_file.read_text().strip().split("\n") return [line for line in lines if not line.startswith("#")] readme = here / "README.rst" readme_content = readme.read_text(encoding="utf-8") changelog = here / "CHANGELOG.rst" changelog_content = changelog.read_text(encoding="utf-8") long_description = readme_content + "\n\n" + changelog_content INSTALL_REQUIRES = _get_dependencies( requirements_file=Path("requirements.txt"), ) DEV_REQUIRES = _get_dependencies( requirements_file=Path("test-requirements.txt"), ) setup( name="pip_check_reqs", version=__version__, description=( "Find packages that should or should not be in requirements for a " "project" ), long_description=long_description, url="https://github.com/r1chardj0n3s/pip-check-reqs", author="Richard Jones", author_email="r1chardj0n3s@gmail.com", maintainer="Adam Dangoor", maintainer_email="adamdangoor@gmail.com", license="MIT", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Topic :: Software Development :: Build Tools", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], python_requires=">=3.9.0", packages=["pip_check_reqs"], entry_points={ "console_scripts": [ "pip-missing-reqs=pip_check_reqs.find_missing_reqs:main", "pip-extra-reqs=pip_check_reqs.find_extra_reqs:main", ], }, install_requires=INSTALL_REQUIRES, extras_require={"dev": DEV_REQUIRES}, ) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694515069.0 pip_check_reqs-2.5.3/test-requirements.txt0000644000076500000240000000021614500037575017573 0ustar00adamstaffblack ruff mypy pyenchant pylint pyright pytest pytest-cov types-setuptools ruamel.yaml # This is used in a test, but not directly imported. ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1694521663.4774625 pip_check_reqs-2.5.3/tests/0000755000076500000240000000000014500054477014475 5ustar00adamstaff././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694521615.0 pip_check_reqs-2.5.3/tests/test_common.py0000644000076500000240000002546614500054417017405 0ustar00adamstaff"""Tests for `common.py`.""" from __future__ import annotations import logging import platform import re import sys import textwrap import types import uuid from pathlib import Path import __main__ import pytest from pip_check_reqs import __version__, common @pytest.mark.parametrize( ("path", "result"), [ (Path("/"), None), (Path("/ham/spam/other.py"), None), (Path("/ham/spam"), None), # a top-level file like this has no package path (Path("__init__.py"), None), (Path("/__init__.py"), None), # no package name (Path("spam/__init__.py"), Path("spam")), (Path("spam/__init__.pyc"), Path("spam")), (Path("spam/__init__.pyo"), Path("spam")), (Path("ham/spam/__init__.py"), Path("ham/spam")), (Path("/ham/spam/__init__.py"), Path("/ham/spam")), ], ) def test_package_path(path: Path, result: Path) -> None: assert common.package_path(path=path) == result, path def test_found_module() -> None: found_module = common.FoundModule(modname="spam", filename=Path("ham")) assert found_module.modname == "spam" assert found_module.filename == Path("ham").resolve() assert not found_module.locations def test_pyfiles_file(tmp_path: Path) -> None: python_file = tmp_path / "example.py" python_file.touch() assert list(common.pyfiles(root=python_file)) == [python_file] def test_pyfiles_file_no_dice(tmp_path: Path) -> None: not_python_file = tmp_path / "example" not_python_file.touch() with pytest.raises( expected_exception=ValueError, match=re.escape( f"{not_python_file} is not a python file or directory", ), ): list(common.pyfiles(root=not_python_file)) def test_pyfiles_package(tmp_path: Path) -> None: python_file = tmp_path / "example.py" nested_python_file = tmp_path / "subdir" / "example.py" not_python_file = tmp_path / "example" python_file.touch() nested_python_file.parent.mkdir() nested_python_file.touch() not_python_file.touch() assert list(common.pyfiles(root=tmp_path)) == [ python_file, nested_python_file, ] @pytest.mark.parametrize( argnames=("statement", "expected_module_names"), argvalues=[ pytest.param("import ast", {"ast"}), pytest.param("import ast, pathlib", {"ast", "pathlib"}), pytest.param("from pathlib import Path", {"pathlib"}), pytest.param("from string import hexdigits", {"string"}), pytest.param("import urllib.request", {"urllib"}), pytest.param("import spam", set[str](), id="The file we are in"), pytest.param("from .foo import bar", set[str](), id="Relative import"), pytest.param("from . import baz", set[str]()), pytest.param( "import re", {"re"}, id="Useful to confirm that the next test is valid", ), pytest.param( "import typing.re", {"typing"}, id="Submodule has same name as a top-level module", ), ], ) def test_find_imported_modules_simple( statement: str, expected_module_names: set[str], tmp_path: Path, ) -> None: """Test for the basic ability to find imported modules.""" spam = tmp_path / "spam.py" spam.write_text(data=statement) result = common.find_imported_modules( paths=[tmp_path], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ) assert set(result.keys()) == expected_module_names for value in result.values(): assert str(value.filename) not in sys.path assert value.filename.name != "__init__.py" assert value.filename.is_absolute() assert value.filename.exists() def test_find_imported_modules_frozen( tmp_path: Path, ) -> None: """Frozen modules are not included in the result.""" frozen_item_names: list[str] = [] sys_module_items = list(sys.modules.items()) for name, value in sys_module_items: try: spec = value.__spec__ except AttributeError: continue if spec is not None and spec.origin == "frozen": frozen_item_names.append(name) assert ( frozen_item_names ), "This test is only valid if there are frozen modules in sys.modules" spam = tmp_path / "spam.py" statement = f"import {frozen_item_names[0]}" spam.write_text(data=statement) result = common.find_imported_modules( paths=[tmp_path], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ) assert set(result.keys()) == set() @pytest.mark.skipif( condition=platform.system() == "Windows", reason=( "Test not supported on Windows, where __main__.__spec__ is not None" ), ) def test_find_imported_modules_main( tmp_path: Path, ) -> None: # pragma: no cover spam = tmp_path / "spam.py" statement = "import __main__" spam.write_text(data=statement) message = ( "This test is only valid if __main__.__spec__ is None. " "That is not the case when running pytest as 'python -m pytest' " "which modifies sys.modules. " "See https://docs.pytest.org/en/7.1.x/how-to/usage.html#calling-pytest-from-python-code" ) assert __main__.__spec__ is None, message result = common.find_imported_modules( paths=[tmp_path], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ) assert set(result.keys()) == set() def test_find_imported_modules_no_spec(tmp_path: Path) -> None: """Modules without a __spec__ are not included in the result. This is often __main__. However, it is also possible to create a module without a __spec__. We prefer to test with a realistic case, but on Windows under `pytest`, `__main__.__spec__` is not None as `__main__` is replaced by pytest. Therefore we need this test to create a module without a __spec__. """ spam = tmp_path / "spam.py" name = "a" + uuid.uuid4().hex statement = f"import {name}" spam.write_text(data=statement) module = types.ModuleType(name=name) module.__spec__ = None sys.modules[name] = module try: result = common.find_imported_modules( paths=[tmp_path], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ) finally: del sys.modules[name] assert set(result.keys()) == set() def test_find_imported_modules_period(tmp_path: Path) -> None: """Imported modules are found if the package name contains a period. An example of this is the module name `"ruamel.yaml"`. https://pypi.org/project/ruamel.yaml/ In particular, `ruamel.yaml` is in `sys.modules` with a period in the name. """ spam = tmp_path / "spam.py" statement = "import ruamel.yaml" spam.write_text(data=statement) result = common.find_imported_modules( paths=[tmp_path], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ) assert set(result.keys()) == {"ruamel.yaml"} @pytest.mark.parametrize( ("ignore_ham", "ignore_hashlib", "expect", "locs"), [ ( False, False, ["ast", "pathlib", "hashlib", "sys"], [ ("spam.py", 2), ("ham.py", 2), ], ), ( False, True, ["ast", "pathlib", "sys"], [("spam.py", 2), ("ham.py", 2)], ), (True, False, ["ast", "sys"], [("spam.py", 2)]), (True, True, ["ast", "sys"], [("spam.py", 2)]), ], ) def test_find_imported_modules_advanced( *, caplog: pytest.LogCaptureFixture, ignore_ham: bool, ignore_hashlib: bool, expect: list[str], locs: list[tuple[str, int]], tmp_path: Path, ) -> None: root = tmp_path spam = root / "spam.py" ham = root / "ham.py" spam_file_contents = textwrap.dedent( """\ from __future__ import braces import ast, sys from . import friend """, ) ham_file_contents = textwrap.dedent( """\ from pathlib import Path import ast, hashlib """, ) spam.write_text(data=spam_file_contents) ham.write_text(data=ham_file_contents) caplog.set_level(logging.INFO) def ignore_files(path: str) -> bool: if Path(path).name == "ham.py" and ignore_ham: return True return False def ignore_mods(module: str) -> bool: if module == "hashlib" and ignore_hashlib: return True return False result = common.find_imported_modules( paths=[root], ignore_files_function=ignore_files, ignore_modules_function=ignore_mods, ) assert set(result) == set(expect) absolute_locations = result["ast"].locations relative_locations = [ (str(Path(item[0]).relative_to(root)), item[1]) for item in absolute_locations ] assert sorted(relative_locations) == sorted(locs) if ignore_ham: assert caplog.records[0].message == f"ignoring: {ham}" @pytest.mark.parametrize( ("ignore_cfg", "candidate", "result"), [ ([], "spam", False), ([], "ham", False), (["spam"], "spam", True), (["spam"], "spam.ham", False), (["spam"], "eggs", False), (["spam*"], "spam", True), (["spam*"], "spam.ham", True), (["spam*"], "eggs", False), (["spam"], str(Path.cwd() / "spam"), True), ], ) def test_ignorer( *, ignore_cfg: list[str], candidate: str, result: bool, ) -> None: ignorer = common.ignorer(ignore_cfg=ignore_cfg) assert ignorer(candidate) == result def test_find_required_modules(tmp_path: Path) -> None: fake_requirements_file = tmp_path / "requirements.txt" fake_requirements_file.write_text("foobar==1\nbarfoo==2") reqs = common.find_required_modules( ignore_requirements_function=common.ignorer(ignore_cfg=["barfoo"]), skip_incompatible=False, requirements_filename=fake_requirements_file, ) assert reqs == {"foobar"} def test_find_required_modules_env_markers(tmp_path: Path) -> None: fake_requirements_file = tmp_path / "requirements.txt" fake_requirements_file.write_text( 'spam==1; python_version<"2.0"\nham==2;\neggs==3\n', ) reqs = common.find_required_modules( ignore_requirements_function=common.ignorer(ignore_cfg=[]), skip_incompatible=True, requirements_filename=fake_requirements_file, ) assert reqs == {"ham", "eggs"} def test_version_info_shows_version_number() -> None: assert __version__ in common.version_info() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694346740.0 pip_check_reqs-2.5.3/tests/test_find_extra_reqs.py0000644000076500000240000000767414477326764021317 0ustar00adamstaff"""Tests for `find_extra_reqs.py`.""" from __future__ import annotations import logging import textwrap from typing import TYPE_CHECKING import black import pytest from pip_check_reqs import common, find_extra_reqs if TYPE_CHECKING: from pathlib import Path def test_find_extra_reqs(tmp_path: Path) -> None: installed_not_imported_required_package = pytest installed_imported_required_package = black fake_requirements_file = tmp_path / "requirements.txt" fake_requirements_file.write_text( textwrap.dedent( f"""\ not_installed_package_12345==1 {installed_imported_required_package.__name__} {installed_not_imported_required_package.__name__} """, ), ) source_dir = tmp_path / "source" source_dir.mkdir() source_file = source_dir / "source.py" source_file.write_text( textwrap.dedent( f"""\ import pprint import {installed_imported_required_package.__name__} """, ), ) result = find_extra_reqs.find_extra_reqs( requirements_filename=fake_requirements_file, paths=[source_dir], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ignore_requirements_function=common.ignorer(ignore_cfg=[]), skip_incompatible=False, ) expected_result = [ "not-installed-package-12345", installed_not_imported_required_package.__name__, ] assert sorted(result) == sorted(expected_result) def test_main_failure( caplog: pytest.LogCaptureFixture, tmp_path: Path, ) -> None: requirements_file = tmp_path / "requirements.txt" requirements_file.write_text("extra") source_dir = tmp_path / "source" source_dir.mkdir() caplog.set_level(logging.WARN) with pytest.raises(SystemExit) as excinfo: find_extra_reqs.main( arguments=[ "--requirements", str(requirements_file), str(source_dir), ], ) assert excinfo.value.code == 1 assert caplog.records[0].message == "Extra requirements:" assert caplog.records[1].message == f"extra in {requirements_file}" def test_main_no_spec(capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit) as excinfo: find_extra_reqs.main(arguments=[]) expected_code = 2 assert excinfo.value.code == expected_code err = capsys.readouterr().err assert err.endswith("error: no source files or directories specified\n") @pytest.mark.parametrize( ("expected_log_levels", "verbose_cfg", "debug_cfg"), [ ({logging.WARNING}, False, False), ({logging.INFO, logging.WARNING}, True, False), ({logging.DEBUG, logging.INFO, logging.WARNING}, False, True), ({logging.DEBUG, logging.INFO, logging.WARNING}, True, True), ], ) def test_logging_config( caplog: pytest.LogCaptureFixture, expected_log_levels: set[int], tmp_path: Path, *, verbose_cfg: bool, debug_cfg: bool, ) -> None: source_dir = tmp_path / "source" source_dir.mkdir() requirements_file = tmp_path / "requirements.txt" requirements_file.touch() arguments = [str(source_dir), "--requirements", str(requirements_file)] if verbose_cfg: arguments.append("--verbose") if debug_cfg: arguments.append("--debug") find_extra_reqs.main(arguments=arguments) for event in [ (logging.DEBUG, "debug"), (logging.INFO, "info"), (logging.WARN, "warn"), ]: find_extra_reqs.log.log(*event) log_levels = {r.levelno for r in caplog.records} assert log_levels == expected_log_levels def test_main_version(capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit): find_extra_reqs.main(arguments=["--version"]) assert capsys.readouterr().out == common.version_info() + "\n" ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1694467124.0 pip_check_reqs-2.5.3/tests/test_find_missing_reqs.py0000644000076500000240000001034414477702064021620 0ustar00adamstaff"""Tests for `find_missing_reqs.py`.""" from __future__ import annotations import logging import textwrap from pathlib import Path import black import pytest from pip_check_reqs import common, find_missing_reqs def test_find_missing_reqs(tmp_path: Path) -> None: installed_imported_not_required_package = pytest installed_imported_required_package = black fake_requirements_file = tmp_path / "requirements.txt" fake_requirements_file.write_text( textwrap.dedent( f"""\ not_installed_package_12345==1 {installed_imported_required_package.__name__} """, ), ) source_dir = tmp_path / "source" source_dir.mkdir() source_file = source_dir / "source.py" source_file.write_text( textwrap.dedent( f"""\ import pprint import {installed_imported_not_required_package.__name__} import {installed_imported_required_package.__name__} """, ), ) result = find_missing_reqs.find_missing_reqs( requirements_filename=fake_requirements_file, paths=[source_dir], ignore_files_function=common.ignorer(ignore_cfg=[]), ignore_modules_function=common.ignorer(ignore_cfg=[]), ) expected_result = [ ( installed_imported_not_required_package.__name__, [ common.FoundModule( modname=installed_imported_not_required_package.__name__, filename=Path( installed_imported_not_required_package.__file__, ).parent, locations=[(str(source_file), 3)], ), ], ), ] assert result == expected_result def test_main_failure( caplog: pytest.LogCaptureFixture, tmp_path: Path, ) -> None: requirements_file = tmp_path / "requirements.txt" requirements_file.touch() source_dir = tmp_path / "source" source_dir.mkdir() source_file = source_dir / "source.py" # We need to import something which is installed. # We choose `pytest` because we know it is installed. source_file.write_text("import pytest") caplog.set_level(logging.WARN) with pytest.raises(SystemExit) as excinfo: find_missing_reqs.main( arguments=[ "--requirements", str(requirements_file), str(source_dir), ], ) assert excinfo.value.code == 1 assert caplog.records[0].message == "Missing requirements:" assert ( caplog.records[1].message == f"{source_file}:1 dist=pytest module=pytest" ) def test_main_no_spec(capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit) as excinfo: find_missing_reqs.main(arguments=[]) expected_code = 2 assert excinfo.value.code == expected_code err = capsys.readouterr().err assert err.endswith("error: no source files or directories specified\n") @pytest.mark.parametrize( ("verbose_cfg", "debug_cfg", "expected_log_levels"), [ (False, False, {logging.WARNING}), (True, False, {logging.INFO, logging.WARNING}), (False, True, {logging.DEBUG, logging.INFO, logging.WARNING}), (True, True, {logging.DEBUG, logging.INFO, logging.WARNING}), ], ) def test_logging_config( *, caplog: pytest.LogCaptureFixture, verbose_cfg: bool, debug_cfg: bool, expected_log_levels: set[int], tmp_path: Path, ) -> None: source_dir = tmp_path / "source" source_dir.mkdir() arguments = [str(source_dir)] if verbose_cfg: arguments.append("--verbose") if debug_cfg: arguments.append("--debug") find_missing_reqs.main(arguments=arguments) for event in [ (logging.DEBUG, "debug"), (logging.INFO, "info"), (logging.WARN, "warn"), ]: find_missing_reqs.log.log(*event) log_levels = {r.levelno for r in caplog.records} assert log_levels == expected_log_levels def test_main_version(capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit): find_missing_reqs.main(arguments=["--version"]) assert capsys.readouterr().out == common.version_info() + "\n"