python-shelltoolbox_0.2.1+bzr17.orig/ez_setup.py0000644000000000000000000003661511723740100020075 0ustar 00000000000000#!python """Bootstrap distribute installation If you want to use setuptools in your package's setup.py, just include this file in the same directory with it, and add this to the top of your setup.py:: from distribute_setup import use_setuptools use_setuptools() If you want to require a specific version of setuptools, set a download mirror, or use an alternate download directory, you can do so by supplying the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import os import sys import time import fnmatch import tempfile import tarfile from distutils import log try: from site import USER_SITE except ImportError: USER_SITE = None try: import subprocess def _python_cmd(*args): args = (sys.executable,) + args return subprocess.call(args) == 0 except ImportError: # will be used for python 2.3 def _python_cmd(*args): args = (sys.executable,) + args # quoting arguments if windows if sys.platform == 'win32': def quote(arg): if ' ' in arg: return '"%s"' % arg return arg args = [quote(arg) for arg in args] return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 DEFAULT_VERSION = "0.6.14" DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" SETUPTOOLS_FAKED_VERSION = "0.6c11" SETUPTOOLS_PKG_INFO = """\ Metadata-Version: 1.0 Name: setuptools Version: %s Summary: xxxx Home-page: xxx Author: xxx Author-email: xxx License: xxx Description: xxx """ % SETUPTOOLS_FAKED_VERSION def _install(tarball): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # installing log.warn('Installing Distribute') if not _python_cmd('setup.py', 'install'): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') finally: os.chdir(old_wd) def _build_egg(egg, tarball, to_dir): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # building an egg log.warn('Building a Distribute egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) finally: os.chdir(old_wd) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') def _do_download(version, download_base, to_dir, download_delay): egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): tarball = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, tarball, to_dir) sys.path.insert(0, egg) import setuptools setuptools.bootstrap_install_from = egg def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15, no_fake=True): # making sure we use the absolute path to_dir = os.path.abspath(to_dir) was_imported = 'pkg_resources' in sys.modules or \ 'setuptools' in sys.modules try: try: import pkg_resources if not hasattr(pkg_resources, '_distribute'): if not no_fake: _fake_setuptools() raise ImportError except ImportError: return _do_download(version, download_base, to_dir, download_delay) try: pkg_resources.require("distribute>="+version) return except pkg_resources.VersionConflict: e = sys.exc_info()[1] if was_imported: sys.stderr.write( "The required version of distribute (>=%s) is not available,\n" "and can't be installed while this script is running. Please\n" "install a more recent version first, using\n" "'easy_install -U distribute'." "\n\n(Currently using %r)\n" % (version, e.args[0])) sys.exit(2) else: del pkg_resources, sys.modules['pkg_resources'] # reload ok return _do_download(version, download_base, to_dir, download_delay) except pkg_resources.DistributionNotFound: return _do_download(version, download_base, to_dir, download_delay) finally: if not no_fake: _create_fake_setuptools_pkg_info(to_dir) def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay=15): """Download distribute from a specified location and return its filename `version` should be a valid distribute version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen tgz_name = "distribute-%s.tar.gz" % version url = download_base + tgz_name saveto = os.path.join(to_dir, tgz_name) src = dst = None if not os.path.exists(saveto): # Avoid repeated downloads try: log.warn("Downloading %s", url) src = urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = src.read() dst = open(saveto, "wb") dst.write(data) finally: if src: src.close() if dst: dst.close() return os.path.realpath(saveto) def _no_sandbox(function): def __no_sandbox(*args, **kw): try: from setuptools.sandbox import DirectorySandbox if not hasattr(DirectorySandbox, '_old'): def violation(*args): pass DirectorySandbox._old = DirectorySandbox._violation DirectorySandbox._violation = violation patched = True else: patched = False except ImportError: patched = False try: return function(*args, **kw) finally: if patched: DirectorySandbox._violation = DirectorySandbox._old del DirectorySandbox._old return __no_sandbox def _patch_file(path, content): """Will backup the file then patch it""" existing_content = open(path).read() if existing_content == content: # already patched log.warn('Already patched.') return False log.warn('Patching...') _rename_path(path) f = open(path, 'w') try: f.write(content) finally: f.close() return True _patch_file = _no_sandbox(_patch_file) def _same_content(path, content): return open(path).read() == content def _rename_path(path): new_name = path + '.OLD.%s' % time.time() log.warn('Renaming %s into %s', path, new_name) os.rename(path, new_name) return new_name def _remove_flat_installation(placeholder): if not os.path.isdir(placeholder): log.warn('Unkown installation at %s', placeholder) return False found = False for file in os.listdir(placeholder): if fnmatch.fnmatch(file, 'setuptools*.egg-info'): found = True break if not found: log.warn('Could not locate setuptools*.egg-info') return log.warn('Removing elements out of the way...') pkg_info = os.path.join(placeholder, file) if os.path.isdir(pkg_info): patched = _patch_egg_dir(pkg_info) else: patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) if not patched: log.warn('%s already patched.', pkg_info) return False # now let's move the files out of the way for element in ('setuptools', 'pkg_resources.py', 'site.py'): element = os.path.join(placeholder, element) if os.path.exists(element): _rename_path(element) else: log.warn('Could not find the %s element of the ' 'Setuptools distribution', element) return True _remove_flat_installation = _no_sandbox(_remove_flat_installation) def _after_install(dist): log.warn('After install bootstrap.') placeholder = dist.get_command_obj('install').install_purelib _create_fake_setuptools_pkg_info(placeholder) def _create_fake_setuptools_pkg_info(placeholder): if not placeholder or not os.path.exists(placeholder): log.warn('Could not find the install location') return pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) setuptools_file = 'setuptools-%s-py%s.egg-info' % \ (SETUPTOOLS_FAKED_VERSION, pyver) pkg_info = os.path.join(placeholder, setuptools_file) if os.path.exists(pkg_info): log.warn('%s already exists', pkg_info) return log.warn('Creating %s', pkg_info) f = open(pkg_info, 'w') try: f.write(SETUPTOOLS_PKG_INFO) finally: f.close() pth_file = os.path.join(placeholder, 'setuptools.pth') log.warn('Creating %s', pth_file) f = open(pth_file, 'w') try: f.write(os.path.join(os.curdir, setuptools_file)) finally: f.close() _create_fake_setuptools_pkg_info = _no_sandbox(_create_fake_setuptools_pkg_info) def _patch_egg_dir(path): # let's check if it's already patched pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') if os.path.exists(pkg_info): if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): log.warn('%s already patched.', pkg_info) return False _rename_path(path) os.mkdir(path) os.mkdir(os.path.join(path, 'EGG-INFO')) pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') f = open(pkg_info, 'w') try: f.write(SETUPTOOLS_PKG_INFO) finally: f.close() return True _patch_egg_dir = _no_sandbox(_patch_egg_dir) def _before_install(): log.warn('Before install bootstrap.') _fake_setuptools() def _under_prefix(location): if 'install' not in sys.argv: return True args = sys.argv[sys.argv.index('install')+1:] for index, arg in enumerate(args): for option in ('--root', '--prefix'): if arg.startswith('%s=' % option): top_dir = arg.split('root=')[-1] return location.startswith(top_dir) elif arg == option: if len(args) > index: top_dir = args[index+1] return location.startswith(top_dir) if arg == '--user' and USER_SITE is not None: return location.startswith(USER_SITE) return True def _fake_setuptools(): log.warn('Scanning installed packages') try: import pkg_resources except ImportError: # we're cool log.warn('Setuptools or Distribute does not seem to be installed.') return ws = pkg_resources.working_set try: setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools', replacement=False)) except TypeError: # old distribute API setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools')) if setuptools_dist is None: log.warn('No setuptools distribution found') return # detecting if it was already faked setuptools_location = setuptools_dist.location log.warn('Setuptools installation detected at %s', setuptools_location) # if --root or --preix was provided, and if # setuptools is not located in them, we don't patch it if not _under_prefix(setuptools_location): log.warn('Not patching, --root or --prefix is installing Distribute' ' in another location') return # let's see if its an egg if not setuptools_location.endswith('.egg'): log.warn('Non-egg installation') res = _remove_flat_installation(setuptools_location) if not res: return else: log.warn('Egg installation') pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') if (os.path.exists(pkg_info) and _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): log.warn('Already patched.') return log.warn('Patching...') # let's create a fake egg replacing setuptools one res = _patch_egg_dir(setuptools_location) if not res: return log.warn('Patched done.') _relaunch() def _relaunch(): log.warn('Relaunching...') # we have to relaunch the process # pip marker to avoid a relaunch bug if sys.argv[:3] == ['-c', 'install', '--single-version-externally-managed']: sys.argv[0] = 'setup.py' args = [sys.executable] + sys.argv sys.exit(subprocess.call(args)) def _extractall(self, path=".", members=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on directories afterwards. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by getmembers(). """ import copy import operator from tarfile import ExtractError directories = [] if members is None: members = self for tarinfo in members: if tarinfo.isdir(): # Extract directories with a safe mode. directories.append(tarinfo) tarinfo = copy.copy(tarinfo) tarinfo.mode = 448 # decimal for oct 0700 self.extract(tarinfo, path) # Reverse sort directories. if sys.version_info < (2, 4): def sorter(dir1, dir2): return cmp(dir1.name, dir2.name) directories.sort(sorter) directories.reverse() else: directories.sort(key=operator.attrgetter('name'), reverse=True) # Set correct owner, mtime and filemode on directories. for tarinfo in directories: dirpath = os.path.join(path, tarinfo.name) try: self.chown(tarinfo, dirpath) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError: e = sys.exc_info()[1] if self.errorlevel > 1: raise else: self._dbg(1, "tarfile: %s" % e) def main(argv, version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" tarball = download_setuptools() _install(tarball) if __name__ == '__main__': main(sys.argv[1:]) python-shelltoolbox_0.2.1+bzr17.orig/setup.py0000755000000000000000000000134311731615654017406 0ustar 00000000000000#!/usr/bin/env python # # Copyright 2012 Canonical Ltd. This software is licensed under the # GNU General Public License version 3 (see the file LICENSE). import ez_setup ez_setup.use_setuptools() __version__ = '0.2.1' from setuptools import setup setup( name='shelltoolbox', version=__version__, packages=['shelltoolbox'], include_package_data=True, zip_safe=False, maintainer='Launchpad Yellow', description=('Helper functions for interacting with shell commands'), license='GPL v3', url='https://launchpad.net/python-shell-toolbox', classifiers=[ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", "Programming Language :: Python", ], ) python-shelltoolbox_0.2.1+bzr17.orig/shelltoolbox/0000755000000000000000000000000011723253543020402 5ustar 00000000000000python-shelltoolbox_0.2.1+bzr17.orig/tests.py0000644000000000000000000004557411731615654017423 0ustar 00000000000000# Copyright 2012 Canonical Ltd. This software is licensed under the # GNU General Public License version 3 (see the file LICENSE). """Tests for Python shell toolbox.""" __metaclass__ = type import getpass import os from subprocess import CalledProcessError import tempfile import unittest from shelltoolbox import ( apt_get_install, cd, command, DictDiffer, environ, file_append, file_prepend, generate_ssh_keys, get_su_command, get_user_home, get_user_ids, join_command, mkdirs, run, search_file, Serializer, ssh, su, user_exists, ) class TestAptGetInstall(unittest.TestCase): packages = ('package1', 'package2') def _get_caller(self, **kwargs): def caller(*args): for k, v in kwargs.items(): self.assertEqual(v, os.getenv(k)) return caller def test_caller(self): # Ensure the correct command line is passed to caller. cmd = apt_get_install(*self.packages, caller=lambda *args: args) expected = ('apt-get', '-y', 'install') + self.packages self.assertTupleEqual(expected, cmd) def test_non_interactive_dpkg(self): # Ensure dpkg is called in non interactive mode. caller = self._get_caller(DEBIAN_FRONTEND='noninteractive') apt_get_install(*self.packages, caller=caller) def test_env_vars(self): # Ensure apt can be run using custom environment variables. caller = self._get_caller(DEBIAN_FRONTEND='noninteractive', LANG='C') apt_get_install(*self.packages, caller=caller, LANG='C') class TestCdContextManager(unittest.TestCase): def test_cd(self): curdir = os.getcwd() self.assertNotEqual('/var', curdir) with cd('/var'): self.assertEqual('/var', os.getcwd()) self.assertEqual(curdir, os.getcwd()) class TestCommand(unittest.TestCase): def testSimpleCommand(self): # Creating a simple command (ls) works and running the command # produces a string. ls = command('/bin/ls') self.assertIsInstance(ls(), str) def testArguments(self): # Arguments can be passed to commands. ls = command('/bin/ls') self.assertIn('Usage:', ls('--help')) def testMissingExecutable(self): # If the command does not exist, an OSError (No such file or # directory) is raised. bad = command('this command does not exist') with self.assertRaises(OSError) as info: bad() self.assertEqual(2, info.exception.errno) def testError(self): # If the command returns a non-zero exit code, an exception is raised. ls = command('/bin/ls') with self.assertRaises(CalledProcessError): ls('--not a valid switch') def testBakedInArguments(self): # Arguments can be passed when creating the command as well as when # executing it. ll = command('/bin/ls', '-al') self.assertIn('rw', ll()) # Assumes a file is r/w in the pwd. self.assertIn('Usage:', ll('--help')) def testQuoting(self): # There is no need to quote special shell characters in commands. ls = command('/bin/ls') ls('--help', '>') class TestDictDiffer(unittest.TestCase): def testStr(self): a = dict(cow='moo', pig='oink') b = dict(cow='moo', pig='oinkoink', horse='nay') diff = DictDiffer(b, a) s = str(diff) self.assertIn("added: {'horse': None} -> {'horse': 'nay'}", s) self.assertIn("removed: {} -> {}", s) self.assertIn("changed: {'pig': 'oink'} -> {'pig': 'oinkoink'}", s) self.assertIn("unchanged: ['cow']", s) def testStrUnmodified(self): a = dict(cow='moo', pig='oink') diff = DictDiffer(a, a) s = str(diff) self.assertEquals('no changes', s) def testAddedOrChanged(self): a = dict(cow='moo', pig='oink') b = dict(cow='moo', pig='oinkoink', horse='nay') diff = DictDiffer(b, a) expected = set(['horse', 'pig']) self.assertEquals(expected, diff.added_or_changed) class TestEnviron(unittest.TestCase): def test_existing(self): # If an existing environment variable is changed, it is # restored during context cleanup. os.environ['MY_VARIABLE'] = 'foo' with environ(MY_VARIABLE='bar'): self.assertEqual('bar', os.getenv('MY_VARIABLE')) self.assertEqual('foo', os.getenv('MY_VARIABLE')) del os.environ['MY_VARIABLE'] def test_new(self): # If a new environment variable is added, it is removed during # context cleanup. with environ(MY_VAR1='foo', MY_VAR2='bar'): self.assertEqual('foo', os.getenv('MY_VAR1')) self.assertEqual('bar', os.getenv('MY_VAR2')) self.assertIsNone(os.getenv('MY_VAR1')) self.assertIsNone(os.getenv('MY_VAR2')) class BaseCreateFile(object): def create_file(self, content): f = tempfile.NamedTemporaryFile('w', delete=False) f.write(content) f.close() return f class BaseTestFile(BaseCreateFile): base_content = 'line1\n' new_content = 'new line\n' def check_file_content(self, content, filename): self.assertEqual(content, open(filename).read()) class TestFileAppend(unittest.TestCase, BaseTestFile): def test_append(self): # Ensure the new line is correctly added at the end of the file. f = self.create_file(self.base_content) file_append(f.name, self.new_content) self.check_file_content(self.base_content + self.new_content, f.name) def test_existing_content(self): # Ensure nothing happens if the file already contains the given line. content = self.base_content + self.new_content f = self.create_file(content) file_append(f.name, self.new_content) self.check_file_content(content, f.name) def test_new_line_in_file_contents(self): # A new line is automatically added before the given content if it # is not present at the end of current file. f = self.create_file(self.base_content.strip()) file_append(f.name, self.new_content) self.check_file_content(self.base_content + self.new_content, f.name) def test_new_line_in_given_line(self): # A new line is automatically added to the given line if not present. f = self.create_file(self.base_content) file_append(f.name, self.new_content.strip()) self.check_file_content(self.base_content + self.new_content, f.name) def test_non_existent_file(self): # Ensure the file is created if it does not exist. filename = tempfile.mktemp() file_append(filename, self.base_content) self.check_file_content(self.base_content, filename) def test_fragment(self): # Ensure a line fragment is not matched. f = self.create_file(self.base_content) fragment = self.base_content[2:] file_append(f.name, fragment) self.check_file_content(self.base_content + fragment, f.name) class TestFilePrepend(unittest.TestCase, BaseTestFile): def test_prpend(self): # Ensure the new content is correctly prepended at the beginning of # the file. f = self.create_file(self.base_content) file_prepend(f.name, self.new_content) self.check_file_content(self.new_content + self.base_content, f.name) def test_existing_content(self): # Ensure nothing happens if the file already starts with the given # content. content = self.base_content + self.new_content f = self.create_file(content) file_prepend(f.name, self.base_content) self.check_file_content(content, f.name) def test_move_content(self): # If the file contains the given content, but not at the beginning, # the content is moved on top. f = self.create_file(self.base_content + self.new_content) file_prepend(f.name, self.new_content) self.check_file_content(self.new_content + self.base_content, f.name) def test_new_line_in_given_line(self): # A new line is automatically added to the given line if not present. f = self.create_file(self.base_content) file_prepend(f.name, self.new_content.strip()) self.check_file_content(self.new_content + self.base_content, f.name) class TestGenerateSSHKeys(unittest.TestCase): def test_generation(self): # Ensure ssh keys are correctly generated. filename = tempfile.mktemp() generate_ssh_keys(filename) first_line = open(filename).readlines()[0].strip() self.assertEqual('-----BEGIN RSA PRIVATE KEY-----', first_line) pub_content = open(filename + '.pub').read() self.assertTrue(pub_content.startswith('ssh-rsa')) class TestGetSuCommand(unittest.TestCase): def test_current_user(self): # If the su is requested as current user, the arguments are # returned as given. cmd = ('ls', '-l') command = get_su_command(getpass.getuser(), cmd) self.assertSequenceEqual(cmd, command) def test_another_user(self): # Ensure "su" is prepended and arguments are correctly quoted. command = get_su_command('nobody', ('ls', '-l', 'my file')) self.assertSequenceEqual( ('su', 'nobody', '-c', "ls -l 'my file'"), command) class TestGetUserHome(unittest.TestCase): def test_existent(self): # Ensure the real home directory is returned for existing users. self.assertEqual('/root', get_user_home('root')) def test_non_existent(self): # If the user does not exist, return a default /home/[username] home. user = '_this_user_does_not_exist_' self.assertEqual('/home/' + user, get_user_home(user)) class TestGetUserIds(unittest.TestCase): def test_get_user_ids(self): # Ensure the correct uid and gid are returned. uid, gid = get_user_ids('root') self.assertEqual(0, uid) self.assertEqual(0, gid) class TestJoinCommand(unittest.TestCase): def test_normal(self): # Ensure a normal command is correctly parsed. command = 'ls -l' self.assertEqual(command, join_command(command.split())) def test_containing_spaces(self): # Ensure args containing spaces are correctly quoted. args = ('command', 'arg containig spaces') self.assertEqual("command 'arg containig spaces'", join_command(args)) def test_empty(self): # Ensure empty args are correctly quoted. args = ('command', '') self.assertEqual("command ''", join_command(args)) class TestMkdirs(unittest.TestCase): def test_intermediate_dirs(self): # Ensure the leaf directory and all intermediate ones are created. base_dir = tempfile.mktemp(suffix='/') dir1 = tempfile.mktemp(prefix=base_dir) dir2 = tempfile.mktemp(prefix=base_dir) mkdirs(dir1, dir2) self.assertTrue(os.path.isdir(dir1)) self.assertTrue(os.path.isdir(dir2)) def test_existing_dir(self): # If the leaf directory already exists the function returns # without errors. mkdirs('/tmp') def test_existing_file(self): # An `OSError` is raised if the leaf path exists and it is a file. f = tempfile.NamedTemporaryFile('w', delete=False) f.close() with self.assertRaises(OSError): mkdirs(f.name) class TestRun(unittest.TestCase): def testSimpleCommand(self): # Running a simple command (ls) works and running the command # produces a string. self.assertIsInstance(run('/bin/ls'), str) def testStdoutReturned(self): # Running a simple command (ls) works and running the command # produces a string. self.assertIn('Usage:', run('/bin/ls', '--help')) def testCalledProcessErrorRaised(self): # If an error occurs a CalledProcessError is raised with the return # code, command executed, and the output of the command. with self.assertRaises(CalledProcessError) as info: run('ls', '--not a valid switch') exception = info.exception self.assertEqual(2, exception.returncode) self.assertEqual("['ls', '--not a valid switch']", exception.cmd) self.assertIn('unrecognized option', exception.output) def testErrorRaisedStdoutNotRedirected(self): with self.assertRaises(CalledProcessError): run('ls', '--not a valid switch', stdout=None) def testNoneArguments(self): # Ensure None is ignored when passed as positional argument. self.assertIn('Usage:', run('/bin/ls', None, '--help', None)) class TestSearchFile(unittest.TestCase, BaseCreateFile): content1 = 'content1\n' content2 = 'content2\n' def setUp(self): self.filename = self.create_file(self.content1 + self.content2).name def tearDown(self): os.remove(self.filename) def test_grep(self): # Ensure plain text is correctly matched. self.assertEqual(self.content2, search_file('ent2', self.filename)) self.assertEqual(self.content1, search_file('content', self.filename)) def test_no_match(self): # Ensure the function does not return false positives. self.assertIsNone(search_file('no_match', self.filename)) def test_regexp(self): # Ensure the function works with regular expressions. self.assertEqual(self.content2, search_file('\w2', self.filename)) class TestSerializer(unittest.TestCase): def setUp(self): self.path = tempfile.mktemp() self.data = {'key': 'value'} def tearDown(self): if os.path.exists(self.path): os.remove(self.path) def test_serializer(self): # Ensure data is correctly serializied and deserialized. s = Serializer(self.path) s.set(self.data) self.assertEqual(self.data, s.get()) def test_existence(self): # Ensure the file is created only when needed. s = Serializer(self.path) self.assertFalse(s.exists()) s.set(self.data) self.assertTrue(s.exists()) def test_default_value(self): # If the file does not exist, the serializer returns a default value. s = Serializer(self.path) self.assertEqual({}, s.get()) s = Serializer(self.path, default=47) self.assertEqual(47, s.get()) def test_another_serializer(self): # It is possible to use a custom serializer (e.g. pickle). import pickle s = Serializer( self.path, serialize=pickle.dump, deserialize=pickle.load) s.set(self.data) self.assertEqual(self.data, s.get()) class TestSSH(unittest.TestCase): def setUp(self): self.last_command = None def remove_command_options(self, cmd): cmd = list(cmd) del cmd[1:7] return cmd def caller(self, cmd): self.last_command = self.remove_command_options(cmd) def check_last_command(self, expected): self.assertSequenceEqual(expected, self.last_command) def test_current_user(self): # Ensure ssh command is correctly generated for current user. sshcall = ssh('example.com', caller=self.caller) sshcall('ls -l') self.check_last_command(['ssh', 'example.com', '--', 'ls -l']) def test_another_user(self): # Ensure ssh command is correctly generated for a different user. sshcall = ssh('example.com', 'myuser', caller=self.caller) sshcall('ls -l') self.check_last_command(['ssh', 'myuser@example.com', '--', 'ls -l']) def test_ssh_key(self): # The ssh key path can be optionally provided. sshcall = ssh('example.com', key='/tmp/foo', caller=self.caller) sshcall('ls -l') self.check_last_command([ 'ssh', '-i', '/tmp/foo', 'example.com', '--', 'ls -l']) def test_error(self): # If the ssh command exits with an error code, a # `subprocess.CalledProcessError` is raised. sshcall = ssh('example.com', caller=lambda cmd: 1) with self.assertRaises(CalledProcessError): sshcall('ls -l') def test_ignore_errors(self): # If ignore_errors is set to True when executing the command, no error # will be raised, even if the command itself returns an error code. sshcall = ssh('example.com', caller=lambda cmd: 1) sshcall('ls -l', ignore_errors=True) current_euid = os.geteuid() current_egid = os.getegid() current_home = os.environ['HOME'] example_euid = current_euid + 1 example_egid = current_egid + 1 example_home = '/var/lib/example' userinfo = {'example_user': dict( ids=(example_euid, example_egid), home=example_home)} effective_values = dict(uid=current_euid, gid=current_egid) def stub_os_seteuid(value): effective_values['uid'] = value def stub_os_setegid(value): effective_values['gid'] = value class TestSuContextManager(unittest.TestCase): def setUp(self): import shelltoolbox self.os_seteuid = os.seteuid self.os_setegid = os.setegid self.shelltoolbox_get_user_ids = shelltoolbox.get_user_ids self.shelltoolbox_get_user_home = shelltoolbox.get_user_home os.seteuid = stub_os_seteuid os.setegid = stub_os_setegid shelltoolbox.get_user_ids = lambda user: userinfo[user]['ids'] shelltoolbox.get_user_home = lambda user: userinfo[user]['home'] def tearDown(self): import shelltoolbox os.seteuid = self.os_seteuid os.setegid = self.os_setegid shelltoolbox.get_user_ids = self.shelltoolbox_get_user_ids shelltoolbox.get_user_home = self.shelltoolbox_get_user_home def testChange(self): with su('example_user'): self.assertEqual(example_euid, effective_values['uid']) self.assertEqual(example_egid, effective_values['gid']) self.assertEqual(example_home, os.environ['HOME']) def testEnvironment(self): with su('example_user') as e: self.assertEqual(example_euid, e.uid) self.assertEqual(example_egid, e.gid) self.assertEqual(example_home, e.home) def testRevert(self): with su('example_user'): pass self.assertEqual(current_euid, effective_values['uid']) self.assertEqual(current_egid, effective_values['gid']) self.assertEqual(current_home, os.environ['HOME']) def testRevertAfterFailure(self): try: with su('example_user'): raise RuntimeError() except RuntimeError: self.assertEqual(current_euid, effective_values['uid']) self.assertEqual(current_egid, effective_values['gid']) self.assertEqual(current_home, os.environ['HOME']) class TestUserExists(unittest.TestCase): def test_user_exists(self): self.assertTrue(user_exists('root')) self.assertFalse(user_exists('_this_user_does_not_exist_')) if __name__ == '__main__': unittest.main() python-shelltoolbox_0.2.1+bzr17.orig/shelltoolbox/__init__.py0000644000000000000000000004642411774541155022532 0ustar 00000000000000# Copyright 2012 Canonical Ltd. # This file is part of python-shell-toolbox. # # python-shell-toolbox is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by the # Free Software Foundation, version 3 of the License. # # python-shell-toolbox is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License # along with python-shell-toolbox. If not, see . """Helper functions for accessing shell commands in Python.""" __metaclass__ = type __all__ = [ 'apt_get_install', 'bzr_whois', 'cd', 'command', 'DictDiffer', 'environ', 'file_append', 'file_prepend', 'generate_ssh_keys', 'get_su_command', 'get_user_home', 'get_user_ids', 'install_extra_repositories', 'join_command', 'mkdirs', 'run', 'Serializer', 'script_name', 'search_file', 'ssh', 'su', 'user_exists', 'wait_for_page_contents', ] from collections import namedtuple from contextlib import contextmanager from email.Utils import parseaddr import errno import json import operator import os import pipes import pwd import re import subprocess import sys from textwrap import dedent import time import urllib2 Env = namedtuple('Env', 'uid gid home') def apt_get_install(*args, **kwargs): """Install given packages using apt. It is possible to pass environment variables to be set during install using keyword arguments. :raises: subprocess.CalledProcessError """ caller = kwargs.pop('caller', run) debian_frontend = kwargs.pop('DEBIAN_FRONTEND', 'noninteractive') with environ(DEBIAN_FRONTEND=debian_frontend, **kwargs): cmd = ('apt-get', '-y', 'install') + args return caller(*cmd) def bzr_whois(user): """Return full name and email of bzr `user`. Return None if the given `user` does not have a bzr user id. """ with su(user): try: whoami = run('bzr', 'whoami') except (subprocess.CalledProcessError, OSError): return None return parseaddr(whoami) @contextmanager def cd(directory): """A context manager to temporarily change current working dir, e.g.:: >>> import os >>> os.chdir('/tmp') >>> with cd('/bin'): print os.getcwd() /bin >>> print os.getcwd() /tmp """ cwd = os.getcwd() os.chdir(directory) try: yield finally: os.chdir(cwd) def command(*base_args): """Return a callable that will run the given command with any arguments. The first argument is the path to the command to run, subsequent arguments are command-line arguments to "bake into" the returned callable. The callable runs the given executable and also takes arguments that will be appeneded to the "baked in" arguments. For example, this code will list a file named "foo" (if it exists): ls_foo = command('/bin/ls', 'foo') ls_foo() While this invocation will list "foo" and "bar" (assuming they exist): ls_foo('bar') """ def callable_command(*args): all_args = base_args + args return run(*all_args) return callable_command @contextmanager def environ(**kwargs): """A context manager to temporarily change environment variables. If an existing environment variable is changed, it is restored during context cleanup:: >>> import os >>> os.environ['MY_VARIABLE'] = 'foo' >>> with environ(MY_VARIABLE='bar'): print os.getenv('MY_VARIABLE') bar >>> print os.getenv('MY_VARIABLE') foo >>> del os.environ['MY_VARIABLE'] If we are adding environment variables, they are removed during context cleanup:: >>> import os >>> with environ(MY_VAR1='foo', MY_VAR2='bar'): ... print os.getenv('MY_VAR1'), os.getenv('MY_VAR2') foo bar >>> os.getenv('MY_VAR1') == os.getenv('MY_VAR2') == None True """ backup = {} for key, value in kwargs.items(): backup[key] = os.getenv(key) os.environ[key] = value try: yield finally: for key, value in backup.items(): if value is None: del os.environ[key] else: os.environ[key] = value def file_append(filename, line): r"""Append given `line`, if not present, at the end of `filename`. Usage example:: >>> import tempfile >>> f = tempfile.NamedTemporaryFile('w', delete=False) >>> f.write('line1\n') >>> f.close() >>> file_append(f.name, 'new line\n') >>> open(f.name).read() 'line1\nnew line\n' Nothing happens if the file already contains the given `line`:: >>> file_append(f.name, 'new line\n') >>> open(f.name).read() 'line1\nnew line\n' A new line is automatically added before the given `line` if it is not present at the end of current file content:: >>> import tempfile >>> f = tempfile.NamedTemporaryFile('w', delete=False) >>> f.write('line1') >>> f.close() >>> file_append(f.name, 'new line\n') >>> open(f.name).read() 'line1\nnew line\n' The file is created if it does not exist:: >>> import tempfile >>> filename = tempfile.mktemp() >>> file_append(filename, 'line1\n') >>> open(filename).read() 'line1\n' """ if not line.endswith('\n'): line += '\n' with open(filename, 'a+') as f: lines = f.readlines() if line not in lines: if not lines or lines[-1].endswith('\n'): f.write(line) else: f.write('\n' + line) def file_prepend(filename, line): r"""Insert given `line`, if not present, at the beginning of `filename`. Usage example:: >>> import tempfile >>> f = tempfile.NamedTemporaryFile('w', delete=False) >>> f.write('line1\n') >>> f.close() >>> file_prepend(f.name, 'line0\n') >>> open(f.name).read() 'line0\nline1\n' If the file starts with the given `line`, nothing happens:: >>> file_prepend(f.name, 'line0\n') >>> open(f.name).read() 'line0\nline1\n' If the file contains the given `line`, but not at the beginning, the line is moved on top:: >>> file_prepend(f.name, 'line1\n') >>> open(f.name).read() 'line1\nline0\n' """ if not line.endswith('\n'): line += '\n' with open(filename, 'r+') as f: lines = f.readlines() if lines[0] != line: try: lines.remove(line) except ValueError: pass lines.insert(0, line) f.seek(0) f.writelines(lines) def generate_ssh_keys(path, passphrase=''): """Generate ssh key pair, saving them inside the given `directory`. >>> generate_ssh_keys('/tmp/id_rsa') 0 >>> open('/tmp/id_rsa').readlines()[0].strip() '-----BEGIN RSA PRIVATE KEY-----' >>> open('/tmp/id_rsa.pub').read().startswith('ssh-rsa') True >>> os.remove('/tmp/id_rsa') >>> os.remove('/tmp/id_rsa.pub') If either of the key files already exist, generate_ssh_keys() will raise an Exception. Note that ssh-keygen will prompt if the keyfiles already exist, but when we're using it non-interactively it's better to pre-empt that behaviour. >>> with open('/tmp/id_rsa', 'w') as key_file: ... key_file.write("Don't overwrite me, bro!") >>> generate_ssh_keys('/tmp/id_rsa') # doctest: +ELLIPSIS Traceback (most recent call last): Exception: File /tmp/id_rsa already exists... >>> os.remove('/tmp/id_rsa') >>> with open('/tmp/id_rsa.pub', 'w') as key_file: ... key_file.write("Don't overwrite me, bro!") >>> generate_ssh_keys('/tmp/id_rsa') # doctest: +ELLIPSIS Traceback (most recent call last): Exception: File /tmp/id_rsa.pub already exists... >>> os.remove('/tmp/id_rsa.pub') """ if os.path.exists(path): raise Exception("File {} already exists.".format(path)) if os.path.exists(path + '.pub'): raise Exception("File {}.pub already exists.".format(path)) return subprocess.call([ 'ssh-keygen', '-q', '-t', 'rsa', '-N', passphrase, '-f', path]) def get_su_command(user, args): """Return a command line as a sequence, prepending "su" if necessary. This can be used together with `run` when the `su` context manager is not enough (e.g. an external program uses uid rather than euid). run(*get_su_command(user, ['bzr', 'whoami'])) If the su is requested as current user, the arguments are returned as given:: >>> import getpass >>> current_user = getpass.getuser() >>> get_su_command(current_user, ('ls', '-l')) ('ls', '-l') Otherwise, "su" is prepended:: >>> get_su_command('nobody', ('ls', '-l', 'my file')) ('su', 'nobody', '-c', "ls -l 'my file'") """ if get_user_ids(user)[0] != os.getuid(): args = [i for i in args if i is not None] return ('su', user, '-c', join_command(args)) return args def get_user_home(user): """Return the home directory of the given `user`. >>> get_user_home('root') '/root' If the user does not exist, return a default /home/[username] home:: >>> get_user_home('_this_user_does_not_exist_') '/home/_this_user_does_not_exist_' """ try: return pwd.getpwnam(user).pw_dir except KeyError: return os.path.join(os.path.sep, 'home', user) def get_user_ids(user): """Return the uid and gid of given `user`, e.g.:: >>> get_user_ids('root') (0, 0) """ userdata = pwd.getpwnam(user) return userdata.pw_uid, userdata.pw_gid def install_extra_repositories(*repositories): """Install all of the extra repositories and update apt. Given repositories can contain a "{distribution}" placeholder, that will be replaced by current distribution codename. :raises: subprocess.CalledProcessError """ distribution = run('lsb_release', '-cs').strip() # Starting from Oneiric, `apt-add-repository` is interactive by # default, and requires a "-y" flag to be set. assume_yes = None if distribution == 'lucid' else '-y' for repo in repositories: repository = repo.format(distribution=distribution) run('apt-add-repository', assume_yes, repository) run('apt-get', 'clean') run('apt-get', 'update') def join_command(args): """Return a valid Unix command line from `args`. >>> join_command(['ls', '-l']) 'ls -l' Arguments containing spaces and empty args are correctly quoted:: >>> join_command(['command', 'arg1', 'arg containing spaces', '']) "command arg1 'arg containing spaces' ''" """ return ' '.join(pipes.quote(arg) for arg in args) def mkdirs(*args): """Create leaf directories (given as `args`) and all intermediate ones. >>> import tempfile >>> base_dir = tempfile.mktemp(suffix='/') >>> dir1 = tempfile.mktemp(prefix=base_dir) >>> dir2 = tempfile.mktemp(prefix=base_dir) >>> mkdirs(dir1, dir2) >>> os.path.isdir(dir1) True >>> os.path.isdir(dir2) True If the leaf directory already exists the function returns without errors:: >>> mkdirs(dir1) An `OSError` is raised if the leaf path exists and it is a file:: >>> f = tempfile.NamedTemporaryFile( ... 'w', delete=False, prefix=base_dir) >>> f.close() >>> mkdirs(f.name) # doctest: +ELLIPSIS Traceback (most recent call last): OSError: ... """ for directory in args: try: os.makedirs(directory) except OSError as err: if err.errno != errno.EEXIST or os.path.isfile(directory): raise def run(*args, **kwargs): """Run the command with the given arguments. The first argument is the path to the command to run. Subsequent arguments are command-line arguments to be passed. This function accepts all optional keyword arguments accepted by `subprocess.Popen`. """ args = [i for i in args if i is not None] pipe = subprocess.PIPE process = subprocess.Popen( args, stdout=kwargs.pop('stdout', pipe), stderr=kwargs.pop('stderr', pipe), close_fds=kwargs.pop('close_fds', True), **kwargs) stdout, stderr = process.communicate() if process.returncode: exception = subprocess.CalledProcessError( process.returncode, repr(args)) # The output argument of `CalledProcessError` was introduced in Python # 2.7. Monkey patch the output here to avoid TypeErrors in older # versions of Python, still preserving the output in Python 2.7. exception.output = ''.join(filter(None, [stdout, stderr])) raise exception return stdout def script_name(): """Return the name of this script.""" return os.path.basename(sys.argv[0]) def search_file(regexp, filename): """Return the first line in `filename` that matches `regexp`.""" with open(filename) as f: for line in f: if re.search(regexp, line): return line def ssh(location, user=None, key=None, caller=subprocess.call): """Return a callable that can be used to run ssh shell commands. The ssh `location` and, optionally, `user` must be given. If the user is None then the current user is used for the connection. The callable internally uses the given `caller`:: >>> def caller(cmd): ... print tuple(cmd) >>> sshcall = ssh('example.com', 'myuser', caller=caller) >>> root_sshcall = ssh('example.com', caller=caller) >>> sshcall('ls -l') # doctest: +ELLIPSIS ('ssh', '-t', ..., 'myuser@example.com', '--', 'ls -l') >>> root_sshcall('ls -l') # doctest: +ELLIPSIS ('ssh', '-t', ..., 'example.com', '--', 'ls -l') The ssh key path can be optionally provided:: >>> root_sshcall = ssh('example.com', key='/tmp/foo', caller=caller) >>> root_sshcall('ls -l') # doctest: +ELLIPSIS ('ssh', '-t', ..., '-i', '/tmp/foo', 'example.com', '--', 'ls -l') If the ssh command exits with an error code, a `subprocess.CalledProcessError` is raised:: >>> ssh('loc', caller=lambda cmd: 1)('ls -l') # doctest: +ELLIPSIS Traceback (most recent call last): CalledProcessError: ... If ignore_errors is set to True when executing the command, no error will be raised, even if the command itself returns an error code. >>> sshcall = ssh('loc', caller=lambda cmd: 1) >>> sshcall('ls -l', ignore_errors=True) """ sshcmd = [ 'ssh', '-t', '-t', # Yes, this second -t is deliberate. See `man ssh`. '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', ] if key is not None: sshcmd.extend(['-i', key]) if user is not None: location = '{}@{}'.format(user, location) sshcmd.extend([location, '--']) def _sshcall(cmd, ignore_errors=False): command = sshcmd + [cmd] retcode = caller(command) if retcode and not ignore_errors: raise subprocess.CalledProcessError(retcode, ' '.join(command)) return _sshcall @contextmanager def su(user): """A context manager to temporarily run the script as a different user.""" uid, gid = get_user_ids(user) os.setegid(gid) os.seteuid(uid) home = get_user_home(user) with environ(HOME=home): try: yield Env(uid, gid, home) finally: os.setegid(os.getgid()) os.seteuid(os.getuid()) def user_exists(username): """Return True if given `username` exists, e.g.:: >>> user_exists('root') True >>> user_exists('_this_user_does_not_exist_') False """ try: pwd.getpwnam(username) except KeyError: return False return True def wait_for_page_contents(url, contents, timeout=120, validate=None): if validate is None: validate = operator.contains start_time = time.time() while True: try: stream = urllib2.urlopen(url) except (urllib2.HTTPError, urllib2.URLError): pass else: page = stream.read() if validate(page, contents): return page if time.time() - start_time >= timeout: raise RuntimeError('timeout waiting for contents of ' + url) time.sleep(0.1) class DictDiffer: """ Calculate the difference between two dictionaries as: (1) items added (2) items removed (3) keys same in both but changed values (4) keys same in both and unchanged values """ # Based on answer by hughdbrown at: # http://stackoverflow.com/questions/1165352 def __init__(self, current_dict, past_dict): self.current_dict = current_dict self.past_dict = past_dict self.set_current = set(current_dict) self.set_past = set(past_dict) self.intersect = self.set_current.intersection(self.set_past) @property def added(self): return self.set_current - self.intersect @property def removed(self): return self.set_past - self.intersect @property def changed(self): return set(key for key in self.intersect if self.past_dict[key] != self.current_dict[key]) @property def unchanged(self): return set(key for key in self.intersect if self.past_dict[key] == self.current_dict[key]) @property def modified(self): return self.current_dict != self.past_dict @property def added_or_changed(self): return self.added.union(self.changed) def _changes(self, keys): new = {} old = {} for k in keys: new[k] = self.current_dict.get(k) old[k] = self.past_dict.get(k) return "%s -> %s" % (old, new) def __str__(self): if self.modified: s = dedent("""\ added: %s removed: %s changed: %s unchanged: %s""") % ( self._changes(self.added), self._changes(self.removed), self._changes(self.changed), list(self.unchanged)) else: s = "no changes" return s class Serializer: """Handle JSON (de)serialization.""" def __init__(self, path, default=None, serialize=None, deserialize=None): self.path = path self.default = default or {} self.serialize = serialize or json.dump self.deserialize = deserialize or json.load def exists(self): return os.path.exists(self.path) def get(self): if self.exists(): with open(self.path) as f: return self.deserialize(f) return self.default def set(self, data): with open(self.path, 'w') as f: self.serialize(data, f)