dirtbike-0.3/0000775000175000017500000000000012655452344013445 5ustar barrybarry00000000000000dirtbike-0.3/setup.py0000664000175000017500000000103412655216343015152 0ustar barrybarry00000000000000#!/usr/bin/env python from setuptools import setup setup(name='dirtbike', version='0.3', description=( 'Convert already-installed Python modules ("distribution") to wheel' ), author='Asheesh Laroia', author_email='asheesh@asheesh.org', url='https://github.com/paulproteus/dirtbike', packages=['dirtbike'], install_requires=[ 'wheel', ], entry_points={ 'console_scripts': [ 'dirtbike = dirtbike.__main__:main', ], }, ) dirtbike-0.3/setup.cfg0000664000175000017500000000007312655452344015266 0ustar barrybarry00000000000000[egg_info] tag_build = tag_svn_revision = 0 tag_date = 0 dirtbike-0.3/AUTHORS.rst0000664000175000017500000000027512651447505015327 0ustar barrybarry00000000000000============================= People who work on dirtbike ============================= * Alex Gaynor * Asheesh Laroia * Barry Warsaw * Your name here, if you like! Help always welcome. dirtbike-0.3/dirtbike.egg-info/0000775000175000017500000000000012655452344016734 5ustar barrybarry00000000000000dirtbike-0.3/dirtbike.egg-info/SOURCES.txt0000664000175000017500000000054512655452344020624 0ustar barrybarry00000000000000AUTHORS.rst DEVELOP.rst MANIFEST.in README.rst mkchroot.sh rmchroot.sh setup.py tox.ini unittest.cfg dirtbike/__init__.py dirtbike/__main__.py dirtbike/strategy.py dirtbike.egg-info/PKG-INFO dirtbike.egg-info/SOURCES.txt dirtbike.egg-info/dependency_links.txt dirtbike.egg-info/entry_points.txt dirtbike.egg-info/requires.txt dirtbike.egg-info/top_level.txtdirtbike-0.3/dirtbike.egg-info/entry_points.txt0000664000175000017500000000006512655452344022233 0ustar barrybarry00000000000000[console_scripts] dirtbike = dirtbike.__main__:main dirtbike-0.3/dirtbike.egg-info/PKG-INFO0000664000175000017500000000044212655452344020031 0ustar barrybarry00000000000000Metadata-Version: 1.0 Name: dirtbike Version: 0.3 Summary: Convert already-installed Python modules ("distribution") to wheel Home-page: https://github.com/paulproteus/dirtbike Author: Asheesh Laroia Author-email: asheesh@asheesh.org License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN dirtbike-0.3/dirtbike.egg-info/requires.txt0000664000175000017500000000000612655452344021330 0ustar barrybarry00000000000000wheel dirtbike-0.3/dirtbike.egg-info/dependency_links.txt0000664000175000017500000000000112655452344023002 0ustar barrybarry00000000000000 dirtbike-0.3/dirtbike.egg-info/top_level.txt0000664000175000017500000000001112655452344021456 0ustar barrybarry00000000000000dirtbike dirtbike-0.3/dirtbike/0000775000175000017500000000000012655452344015242 5ustar barrybarry00000000000000dirtbike-0.3/dirtbike/__main__.py0000664000175000017500000000231212652326663017333 0ustar barrybarry00000000000000import os import argparse from . import make_wheel_file def parseargs(): parser = argparse.ArgumentParser('Turn OS packages into wheels') parser.add_argument('-d', '--directory', help="""Leave the new .whl file in the given directory. Otherwise the default is to use the current working directory. If DIRECTORY doesn't exist, it will be created. This overrides $DIRTBIKE_DIRECTORY""", default=os.environ.get('DIRTBIKE_DIRECTORY')) parser.add_argument('package', nargs=1, help="""The name of the package to rewheel, as seen by Python (not your OS!).""") parser.epilog = """\ dirtbike also recognizes the environment variable $DIRTBIKE_DIRECTORY which if set, is used as the directory to put .whl files in. This is analogous to the -d/--directory option, although the command line switch takes precedence.""" return parser.parse_args() def main(): args = parseargs() # For convenience and readability of make_wheel_file(). args.package = args.package[0] make_wheel_file(args) if __name__ == '__main__': main() dirtbike-0.3/dirtbike/__init__.py0000664000175000017500000001471112655216426017356 0ustar barrybarry00000000000000from __future__ import ( absolute_import, division, print_function, unicode_literals, ) import os import errno import atexit import shutil import tempfile import distutils.dist import wheel.bdist_wheel from distutils.command.install_egg_info import install_egg_info from glob import glob try: from unittest.mock import patch except ImportError: # Python 2. from mock import patch from .strategy import ( DpkgEggStrategy, DpkgImpStrategy, DpkgImportCalloutStrategy, DpkgImportlibStrategy, WheelStrategy) STRATEGIES = ( # The order is significant here, so DO NOT sort alphabetically. WheelStrategy, DpkgEggStrategy, DpkgImportlibStrategy, DpkgImpStrategy, DpkgImportCalloutStrategy, ) # os.makedirs(, exist_ok=True) doesn't exist in Python 2. def _mkdir_p(dirname): if not dirname: raise ValueError("I refuse to operate on false-y values.") try: os.makedirs(dirname) except OSError as e: if e.errno != errno.EEXIST: raise def _copy_file_making_dirs_as_needed(src, dst): _mkdir_p(os.path.dirname(dst)) shutil.copy(src, dst) def make_wheel_file(args): distribution_name = args.package # Grab the metadata for the installed version of this distribution. for strategy_class in STRATEGIES: strategy = strategy_class(distribution_name) if strategy.can_succeed: break else: raise RuntimeError( 'No strategy for finding package contents: {}'.format( distribution_name)) assert strategy.files is not None # Create a Distribution object so that the wheel.bdist_wheel machinery can # operate happily. We copy any metadata we need out of the installed # package metadata into this thing. dummy_dist_distribution_obj = distutils.dist.Distribution(attrs={ 'name': strategy.name, 'version': strategy.version, }) # The wheel generator will clean up this directory when it exits, but # let's just be sure that any failures don't leave this directory. bdist_dir = tempfile.mkdtemp() dist_dir = tempfile.mkdtemp() if os.environ.get('DIRTBIKE_KEEP_TEMP'): print('Keeping pre-zip staging directory', bdist_dir) print('Keeping post-zip temporary directory', dist_dir) else: atexit.register(shutil.rmtree, bdist_dir, ignore_errors=True) atexit.register(shutil.rmtree, dist_dir, ignore_errors=True) wheel_generator = wheel.bdist_wheel.bdist_wheel( dummy_dist_distribution_obj) wheel_generator.universal = True wheel_generator.bdist_dir = bdist_dir wheel_generator.dist_dir = dist_dir for filename in strategy.files: # The list of files sometimes contains the empty string. That's not # much of a file, so we don't bother adding it to the archive. if len(filename) == 0: continue # NOTE: If a file is not in the "location" that the installed package # supposedly lives in, we skip it. This means we are likely to skip # console scripts. if filename.startswith('/'): abspath = os.path.abspath(filename) else: abspath = os.path.abspath( os.path.join(strategy.location, filename)) found = False is_file = False if abspath.startswith(strategy.location) and os.path.exists(abspath): found = True is_file = os.path.isfile(abspath) # Print a warning in the case that some file is missing, then skip it. if not found: print('Skipping', abspath, 'because we could not find it in the metadata location.') continue # Skip directories. if found and not is_file: continue # Skip the dist-info directory, since bdist_wheel will # recreate it for us. if '.dist-info' in abspath: continue # Skip any *.pyc files or files that are in a __pycache__ directory. if '__pycache__' in abspath or abspath.endswith('.pyc'): continue _copy_file_making_dirs_as_needed( abspath, os.path.abspath(bdist_dir + '/' + filename)) # This is all a bit of a mess, but as they say "if it's # distutils/setuptools, evil begets evil". Here's what's going on. # # Issue #19 describes a problem where the entry_points.txt file doesn't # survive from the .egg-info directory into the .dist-info directory in # the resulting wheel. This is because bdist_wheel called # install_egg_info which first deletes the entire .egg-info directory! # # The only way to prevent this is to monkey patch install_egg_info so it # doesn't run. We already (probably) have a good .egg-info directory # anyway, so it's not needed. # # This being distutils, there are of course complications. First and # easiest is that we might not have an .egg-info directory. This can # happen if we're rewheeling some stdlib package, e.g. ipaddress. It can # also happen in Debian for split packages, such as pkg_resources, which # is really part of setuptools, but is provided in a separate .deb and # doesn't have an .egg-info. # # But the bdist_wheel machinery *requires* an egg-info, so when it's # missing, let the install_egg_info command actually run. We don't care # too much because we know there won't be an entry_points.txt file for the # package, but it makes this stack of hack happy. egg_infos = glob(os.path.join(bdist_dir, '*.egg-info')) assert len(egg_infos) <= 1, egg_infos wheel_generator.egginfo_dir = ( egg_infos[0] if len(egg_infos) > 0 else None) # Call finalize_options() to tell bdist_wheel we are done playing with # metadata. wheel_generator.finalize_options() # We can do this more elegantly when we're Python 3-only. if wheel_generator.egginfo_dir is None: wheel_generator.run() else: # Using mock's patch.object() is the most reliable way to monkey patch # away the install-egg-info command. with patch.object(install_egg_info, 'run'): wheel_generator.run() # Move the resulting .whl to its final destination. files = glob(os.path.join(dist_dir, '{}*.whl'.format(distribution_name))) assert len(files) == 1, files destination = ( os.getcwd() if args.directory is None else args.directory) _mkdir_p(destination) shutil.move(files[0], destination) dirtbike-0.3/dirtbike/strategy.py0000664000175000017500000002352512652303562017457 0ustar barrybarry00000000000000from __future__ import ( absolute_import, division, print_function, unicode_literals, ) import os import imp import sys import errno import importlib import subprocess import pkg_resources def _abspathify(filenames, location): paths = [] for filename in filenames: # The list of files sometimes contains the empty string. That's not # much of a file, so we don't bother adding it to the archive. if len(filename) == 0: continue # NOTE: If a file is not in the "location" that the installed package # supposedly lives in, we skip it. This means we are likely to skip # console scripts. if filename.startswith('/'): abspath = os.path.abspath(filename) else: abspath = os.path.abspath(os.path.join(location, filename)) found = False is_file = False if abspath.startswith(location) and os.path.exists(abspath): found = True is_file = os.path.isfile(abspath) # Print a warning in the case that some file is missing, then skip it. if not found: print('Skipping', abspath, 'because we could not find it in the metadata location.') continue # Skip directories. if found and not is_file: continue # Skip the dist-info directory, since bdist_wheel will # recreate it for us. if '.dist-info' in abspath: continue # Skip any *.pyc files or files that are in a __pycache__ directory. if '__pycache__' in abspath or abspath.endswith('.pyc'): continue paths.append(abspath) return paths class Strategy(object): """Encapsulation of a distribution's contents strategies.""" def __init__(self, name): self._name = name @property def name(self): """The project's name.""" return self._name @property def can_succeed(self): """A boolean which describes whether this strategy can succeed.""" raise NotImplementedError @property def version(self): """The version associated with the installed package.""" @property def files(self): """A list of files contained in the package or None. If this strategy cannot find the named package's contents, this attribute will be None. """ raise NotImplementedError @property def location(self): """The metadata location.""" raise NotImplementedError class WheelStrategy(Strategy): """Use wheel metadata to find package contents.""" def __init__(self, name): super(WheelStrategy, self).__init__(name) self._files = None try: self._metadata = pkg_resources.get_distribution(self._name) except pkg_resources.DistributionNotFound: return try: # If we're lucky, the information for what files are installed on # the system are available in RECORD, aka wheel metadata. files = self._metadata.get_metadata('RECORD').splitlines() # Python 3 - use FileNotFoundError except IOError as error: self._files = None # Let's find the path to an egg-info file and ask dpkg for the # file metadata. if error.errno == errno.ENOENT: return raise self._files = _abspathify(files, self._metadata.location) @property def can_succeed(self): return self._files is not None @property def version(self): return self._metadata.version @property def files(self): return self._files @property def name(self): return self._metadata.project_name @property def location(self): return self._metadata.location class _DpkgBaseStrategy(object): def _find_files(self, path_to_some_file, relative_to): stdout = subprocess.check_output( ['/usr/bin/dpkg', '-S', path_to_some_file], universal_newlines=True) pkg_name, colon, path = stdout.partition(':') stdout = subprocess.check_output( ['/usr/bin/dpkg', '-L', pkg_name], universal_newlines=True) # Now we have all the files from the Debian package. However, # RECORD-style files lists are all relative to the site-packages # directory in which the package was installed. for filename in stdout.splitlines(): if filename.startswith(relative_to): shortened_filename = filename[len(relative_to):] if len(shortened_filename) == 0: continue if shortened_filename.startswith('/'): shortened_filename = shortened_filename[1:] yield shortened_filename class DpkgEggStrategy(Strategy, _DpkgBaseStrategy): """Use Debian-specific strategy for finding a package's contents.""" # It would be nice to be able to remove the Debian-specific code so that # this can rely entirely in the pip pseudo-standard of # "installed-files.txt" etc. However, packages in Debian testing do not # yet distribute an installed-files.txt, so probably it is useful to have # the Debian-specific code in this version. # # To be semver-esque, a version with the Debian-specific code # removed would presumably have a bumped "major" version number. def __init__(self, name): super(DpkgEggStrategy, self).__init__(name) try: self._metadata = pkg_resources.get_distribution(name) except pkg_resources.DistributionNotFound: self._metadata = None return # Find the .egg-info directory, and then search the dpkg database for # which package provides it. path_to_egg_info = self._metadata._provider.egg_info self._files = list(self._find_files(path_to_egg_info, self._metadata.location)) @property def name(self): return self._metadata.project_name @property def can_succeed(self): return self._metadata is not None @property def version(self): return self._metadata.version @property def files(self): return self._files @property def location(self): return self._metadata.location class DpkgImportlibStrategy(Strategy, _DpkgBaseStrategy): """Use dpkg based on Python 3's importlib.""" def __init__(self, name): super(DpkgImportlibStrategy, self).__init__(name) spec = self._spec = None try: spec = importlib.util.find_spec(name) except AttributeError: # Must be Python 2. pass if spec is None or not spec.has_location: return # I'm not sure what to do if this is a namespace package, so punt. if ( spec.submodule_search_locations is None or len(spec.submodule_search_locations) != 1): return self._spec = spec location = spec.submodule_search_locations[0] # The location will be the package directory, but we need its parent # so that imports will work. This will very likely be # /usr/lib/python3/dist-packages location = self._location = os.path.dirname(location) self._files = list(self._find_files(self._spec.origin, location)) @property def can_succeed(self): return self._spec is not None @property def location(self): return self._location @property def files(self): return self._files class DpkgImpStrategy(Strategy, _DpkgBaseStrategy): """Use dpkg based on Python 2's imp API.""" def __init__(self, name): super(DpkgImpStrategy, self).__init__(name) self._location = None try: filename, pathname, description = imp.find_module(name) except ImportError: return if pathname is None: return # Don't allow a stdlib package to sneak in. path_components = pathname.split(os.sep) if ( 'site-packages' not in path_components and 'dist-packages' not in path_components): return # The location will be the package directory, but we need it's parent # so that imports will work. This will very likely be # /usr/lib/python2.7/dist-packages location = self._location = os.path.dirname(pathname) self._files = list(self._find_files(pathname, location)) @property def can_succeed(self): return self._location is not None @property def location(self): return self._location @property def files(self): return self._files class DpkgImportCalloutStrategy(Strategy, _DpkgBaseStrategy): """ Use dpkg, but find the file by shelling out to some other Python.""" def __init__(self, name): super(DpkgImportCalloutStrategy, self).__init__(name) self._location = None other_python = '/usr/bin/python{}'.format( 2 if sys.version_info.major == 3 else 3) try: stdout = subprocess.check_output( [other_python, '-c', 'import {0}; print({0}.__file__)'.format(name)], universal_newlines=True) except subprocess.CalledProcessError: return filename = stdout.splitlines()[0] # In Python 2, this will end with .pyc but that's not owned by any # package. So ensure the path ends in .py always. root, ext = os.path.splitext(filename) self._location = os.path.dirname(filename) self._files = list(self._find_files(root + '.py', self._location)) @property def can_succeed(self): return self._location is not None @property def location(self): return self._location @property def files(self): return self._files dirtbike-0.3/MANIFEST.in0000664000175000017500000000023612652456650015205 0ustar barrybarry00000000000000include *.py MANIFEST.in *.ini *.cfg *.sh global-include *.txt *.rst exclude .gitignore prune build prune dirtbike/tests/example/stupid prune .tox prune dist dirtbike-0.3/unittest.cfg0000664000175000017500000000022412651447505016002 0ustar barrybarry00000000000000[unittest] verbose = 2 plugins = dirtbike.testing.nose nose2.plugins.layers [dirtbike] always-on = True [log-capture] always-on = False dirtbike-0.3/DEVELOP.rst0000664000175000017500000000627712652432043015300 0ustar barrybarry00000000000000===================== Developing dirtbike ===================== dirtbike is maintained on `GitHub `__ For now, you're only going to be able to run and test dirtbike on a Debian (or derivative) system. For porting to other distributions, contributions are welcome! To run the test suite, you'll need to ``apt-get install`` the following packages: * debootstrap * dpkg-dev * lsb-release * python * python-stdeb * python-wheel * python3 * python3-stdeb * python3-wheel * schroot * tox And probably more stuff I'm forgetting. Depending on your Debian version, you might have to ``apt-get install python3.5`` manually. Setting things up ================= dirtbike's test suite installs debs that it creates, and you really don't want to be sudo-messing with your development system. For this, the test suite relies on the existence of a schroot environment, which you have to manually create first. We provide some useful scripts for you though. You only need to do this once: $ sudo ./mkchroot.sh This creates an overlay schroot named ``dirtbike--`` where *distro* is the code name of your distribution (e.g. ``unstable``, ``xenial``), and *arch* is your host's architecture (e.g. ``amd64``). Thus, after running the ``mkchroot.sh`` command, running ``schroot -l`` should list something like ``dirtbike-xenial-amd64``. The stupid project ================== The test suite uses a simple pure-Python project pulled in as a git submodule. Be sure to do this once after you clone this repository, if you didn't already do ``git clone --recursive``. $ git submodule init $ git submodule update Tearing things down =================== It's fine to leave the dirtbike schroot hanging around. You might be interested in Barry Warsaw's `chup `__ script for easily keeping all your chroots up-to-date. If you want to clean your system up, just run: $ sudo ./rmchroot.sh which of course deletes the dirtbike schroot directory and configuration file. If later you want to run the test suite again, you'll have to recreate the schroot with the ``mkchroot.sh`` script. Running the tests ================= You should be able to run the test suite against all supported and installed versions of Python (currently, 2.7, 3.4, and 3.5) just by running: $ tox If you want to isolate a single test, you can do it like this: $ .tox/py35/bin/python -m nose2 -vv -P This only runs the test suite against Python 3.5, and it only runs tests matching the given *pattern*, which is just a Python regular expression. Notes ===== Generally, subcommands which are overly verbose have most of their spew suppressed. You can see the gory details if you set the environment variable ``DIRTBIKE_DEBUG`` to any non-empty value. If you want to keep the schroot sessions around after the test suite finishes, set the environment variable ``DIRTBIKE_DEBUG_SESSIONS`` to any non-empty value. The session ids will be printed, and it's up to you to end them explicitly. Note that multiple new, randomly named sessions may be created. You can destroy them all all quickly with ``schroot -e --all-sessions``. dirtbike-0.3/mkchroot.sh0000775000175000017500000000313612655216426015634 0ustar barrybarry00000000000000#!/bin/bash set -euo pipefail CH_ARCH=${CH_ARCH:-`dpkg-architecture -q DEB_HOST_ARCH`} CH_DISTRO=${CH_DISTRO:-`lsb_release -cs`} CH_VENDOR=${CH_VENDOR:-`lsb_release -is`} CH_GROUPS=${CH_GROUPS:-"sbuild,root"} CHROOT=dirtbike-$CH_DISTRO-$CH_ARCH CHROOT_DIR=/var/lib/schroot/chroots/$CHROOT INCLUDES=eatmydata,gdebi-core,software-properties-common,python3-all if [ "$CH_VENDOR" = "Ubuntu" ] then UNIONTYPE=overlayfs else UNIONTYPE=overlay fi echo "Creating schroot $CHROOT for $CH_GROUPS" cat > /etc/schroot/chroot.d/$CHROOT<`__. That is an admittedly strange goal. The deeper purpose is to help Debian packages like `pip` vendor their dependencies in a way compatible with the packaging policy for Debian, and hopefully other GNU/Linux distributions. Therefore, we am eager to see this tool discussed and/or adopted by Fedora, Debian, Ubuntu, and any other software distributions that distribute pip as well as other Python packages. License ======= This software is available under the terms of the same license as `pya/pip`:: Copyright (c) 2008-2014 The developers (see AUTHORS.txt file) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Origin of the name ================== The name comes from a They Might Be Giants song:: Here comes the dirt bike, Beware of the dirt bike. [...] You see I never thought I'd understand. Till that bike took me by the hand, Now I ride. The real motivation behind the name was to find a word or two that alludes to the idea of "wheel." "Boat of Car" might have been too confusing. dirtbike-0.3/tox.ini0000664000175000017500000000123112655216426014754 0ustar barrybarry00000000000000[tox] envlist = py27,py34,py35 [testenv] commands = python -m nose2 -v deps = nose2 pip stdeb wheel py27: mock usedevelop = True passenv = DIRTBIKE_* CH_* [coverage] rcfile = {toxinidir}/coverage.ini rc = --rcfile={[coverage]rcfile} [testenv:coverage] basepython = python3.4 commands = coverage run {[coverage]rc} -m nose2 -v coverage combine {[coverage]rc} coverage html {[coverage]rc} #sitepackages = True usedevelop = True whitelist_externals = python-coverage deps = nose2 coverage setenv = COVERAGE_PROCESS_START={[coverage]rcfile} COVERAGE_OPTIONS="-p" COVERAGE_FILE={toxinidir}/.coverage dirtbike-0.3/rmchroot.sh0000775000175000017500000000053312651447505015641 0ustar barrybarry00000000000000#!/bin/bash set -euo pipefail CH_ARCH=${CH_ARCH:-`dpkg-architecture -q DEB_HOST_ARCH`} CH_DISTRO=${CH_DISTRO:-`lsb_release -cs`} CH_VENDOR=${CH_VENDOR:-`lsb_release -is`} CH_GROUPS=${CH_GROUPS:-"sbuild,root"} CHROOT=dirtbike-$CH_DISTRO-$CH_ARCH CHROOT_DIR=/var/lib/schroot/chroots/$CHROOT rm -rf $CHROOT_DIR rm -f /etc/schroot/chroot.d/$CHROOT