piuparts-0.56/ 0000775 0000000 0000000 00000000000 12323207437 010206 5 ustar piuparts-0.56/piuparts.py 0000664 0000000 0000000 00000360031 12323207374 012432 0 ustar #!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2005 Lars Wirzenius (liw@iki.fi)
# Copyright © 2010-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
"""Debian package installation and uninstallation tester.
This program sets up a minimal Debian system in a chroot, and installs
and uninstalls packages and their dependencies therein, looking for
problems.
See the manual page (piuparts.1, generated from piuparts.1.txt) for
more usage information.
Lars Wirzenius
"""
VERSION = "__PIUPARTS_VERSION__"
import time
import logging
import optparse
import sys
import commands
import tempfile
import shutil
import os
import tarfile
import stat
import re
import pickle
import subprocess
import unittest
import urllib
import uuid
from signal import alarm, signal, SIGALRM, SIGTERM, SIGKILL
try:
from debian import deb822
except ImportError:
from debian_bundle import deb822
import piupartslib.conf
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
class Defaults:
"""Default settings which depend on flavor of Debian.
Some settings, such as the default mirror and distribution, depend on
which flavor of Debian we run under: Debian itself, or a derived
distribution such as Ubuntu. This class abstracts away the defaults
so that the rest of the code can just refer to the values defined
herein.
"""
def get_components(self):
"""Return list of default components for a mirror."""
def get_mirror(self):
"""Return default mirror."""
def get_distribution(self):
"""Return default distribution."""
def get_keyring(self):
"""Return default keyring."""
class DebianDefaults(Defaults):
def get_components(self):
return ["main", "contrib", "non-free"]
def get_mirror(self):
return [("http://cdn.debian.net/debian", self.get_components())]
def get_distribution(self):
return ["sid"]
def get_keyring(self):
return "/usr/share/keyrings/debian-archive-keyring.gpg"
class UbuntuDefaults(Defaults):
def get_components(self):
return ["main", "universe", "restricted", "multiverse"]
def get_mirror(self):
return [("http://archive.ubuntu.com/ubuntu", self.get_components())]
def get_distribution(self):
return ["trusty"]
def get_keyring(self):
return "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
class DefaultsFactory:
"""Instantiate the right defaults class."""
def guess_flavor(self):
p = subprocess.Popen(["lsb_release", "-i", "-s"],
stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
return stdout.strip().lower()
def new_defaults(self):
if not settings.defaults:
settings.defaults = self.guess_flavor()
if settings.defaults.lower() == "debian":
return DebianDefaults()
if settings.defaults.lower() == "ubuntu":
return UbuntuDefaults()
logging.error("Unknown set of defaults: %s" % settings.defaults)
panic()
class Settings:
"""Global settings for this program."""
def __init__(self):
self.defaults = None
self.tmpdir = None
self.keep_tmpdir = False
self.max_command_output_size = 3 * 1024 * 1024 # 3 MB (daptup on dist-upgrade)
self.max_command_runtime = 30 * 60 # 30 minutes (texlive-full on dist-upgrade)
self.single_changes_list = False
self.args_are_package_files = True
# distro setup
self.proxy = None
self.debian_mirrors = []
self.extra_repos = []
self.testdebs_repo = None
self.debian_distros = []
self.keep_sources_list = False
self.keyring = None
self.do_not_verify_signatures = False
self.install_recommends = False
self.eatmydata = True
self.dpkg_force_unsafe_io = True
self.dpkg_force_confdef = False
self.scriptsdirs = []
self.bindmounts = []
self.allow_database = False
# chroot setup
self.arch = None
self.basetgz = None
self.savetgz = None
self.lvm_volume = None
self.lvm_snapshot_size = "1G"
self.adt_virt = None
self.existing_chroot = None
self.schroot = None
self.end_meta = None
self.save_end_meta = None
self.skip_minimize = True
self.minimize = False
self.debfoster_options = None
# tests and checks
self.no_install_purge_test = False
self.no_upgrade_test = False
self.distupgrade_to_testdebs = False
self.install_remove_install = False
self.install_purge_install = False
self.list_installed_files = False
self.extra_old_packages = []
self.skip_cronfiles_test = False
self.skip_logrotatefiles_test = False
self.adequate = True
self.check_broken_diversions = True
self.check_broken_symlinks = True
self.warn_broken_symlinks = True
self.warn_on_others = False
self.warn_on_leftovers_after_purge = False
self.warn_on_debsums_errors = False
self.warn_if_inadequate = True
self.pedantic_purge_test = False
self.ignored_files = [
# piuparts state
"/usr/sbin/policy-rc.d",
# system state
"/boot/grub/",
"/etc/X11/",
"/etc/X11/default-display-manager",
"/etc/aliases",
"/etc/aliases.db",
"/etc/crypttab",
"/etc/group",
"/etc/group-",
"/etc/gshadow",
"/etc/gshadow-",
"/etc/hosts",
"/etc/inetd.conf",
"/etc/inittab",
"/etc/ld.so.cache",
"/etc/mailname",
"/etc/mtab",
"/etc/network/interfaces",
"/etc/news/",
"/etc/news/organization",
"/etc/news/server",
"/etc/news/servers",
"/etc/news/whoami",
"/etc/nologin",
"/etc/passwd",
"/etc/passwd-",
"/etc/shadow",
"/etc/shadow-",
"/usr/share/info/dir",
"/usr/share/info/dir.old",
"/var/cache/ldconfig/aux-cache",
"/var/crash/",
"/var/games/",
# package management
"/etc/apt/apt.conf.d/01autoremove-kernels",
"/etc/apt/secring.gpg",
"/etc/apt/trustdb.gpg",
"/etc/apt/trusted.gpg",
"/etc/apt/trusted.gpg~",
"/usr/share/keyrings/debian-archive-removed-keys.gpg~",
"/var/cache/apt/archives/lock",
"/var/cache/apt/pkgcache.bin",
"/var/cache/apt/srcpkgcache.bin",
"/var/cache/debconf/",
"/var/cache/debconf/config.dat",
"/var/cache/debconf/config.dat.old",
"/var/cache/debconf/config.dat-old",
"/var/cache/debconf/passwords.dat",
"/var/cache/debconf/passwords.dat.old",
"/var/cache/debconf/templates.dat",
"/var/cache/debconf/templates.dat.old",
"/var/cache/debconf/templates.dat-old",
"/var/lib/apt/extended_states",
"/var/lib/cdebconf/",
"/var/lib/cdebconf/passwords.dat",
"/var/lib/cdebconf/questions.dat",
"/var/lib/cdebconf/questions.dat-old",
"/var/lib/cdebconf/templates.dat",
"/var/lib/cdebconf/templates.dat-old",
"/var/lib/dpkg/arch",
"/var/lib/dpkg/available",
"/var/lib/dpkg/available-old",
"/var/lib/dpkg/diversions",
"/var/lib/dpkg/diversions-old",
"/var/lib/dpkg/lock",
"/var/lib/dpkg/status",
"/var/lib/dpkg/status-old",
"/var/lib/dpkg/statoverride",
"/var/lib/dpkg/statoverride-old",
"/var/log/alternatives.log",
"/var/log/apt/history.log",
"/var/log/apt/term.log",
"/var/log/bootstrap.log",
"/var/log/dbconfig-common/dbc.log",
"/var/log/dpkg.log",
# system logfiles
"/var/log/auth.log",
"/var/log/daemon.log",
"/var/log/debug",
"/var/log/faillog",
"/var/log/kern.log",
"/var/log/lastlog",
"/var/log/lpr.log",
"/var/log/mail.err",
"/var/log/mail.info",
"/var/log/mail.log",
"/var/log/mail.warn",
"/var/log/messages",
"/var/log/news/",
"/var/log/news/news.crit",
"/var/log/news/news.err",
"/var/log/news/news.notice",
"/var/log/secure",
"/var/log/syslog",
"/var/log/user.log",
# application logfiles
# actually, only modification should be permitted here, but not creation/removal
"/var/log/fontconfig.log",
# home directories of system accounts
"/var/lib/gozerbot/",
"/var/lib/nagios/", # nagios* (#668756)
"/var/lib/onioncat/", # onioncat
"/var/lib/rbldns/",
"/var/spool/powerdns/", # pdns-server (#531134), pdns-recursor (#531135)
# work around broken symlinks
"/usr/lib/python2.6/dist-packages/python-support.pth", #635493 and #385775
"/usr/lib/python2.7/dist-packages/python-support.pth",
# work around #316521 dpkg: incomplete cleanup of empty directories
"/etc/apache2/",
"/etc/apache2/conf.d/",
"/etc/cron.d/",
"/etc/nagios-plugins/config/",
"/etc/php5/",
"/etc/php5/conf.d/",
"/etc/php5/mods-available/",
"/etc/sgml/",
"/etc/ssl/",
"/etc/ssl/certs/",
"/etc/ssl/private/",
"/etc/xml/",
# HACKS
]
self.ignored_patterns = [
# system state
"/dev/.*",
"/etc/init.d/\.depend.*",
"/run/.*",
"/var/backups/.*",
"/var/cache/man/.*",
"/var/mail/.*",
"/var/run/.*",
# package management
"/etc/apt/trusted.gpg.d/.*.gpg~",
"/var/lib/apt/lists/.*",
"/var/lib/dpkg/alternatives/.*",
"/var/lib/dpkg/triggers/.*",
"/var/lib/insserv/run.*.log",
"/var/lib/ucf/.*",
"/var/lib/update-rc.d/.*",
# application data
"/var/lib/citadel/(data/.*)?",
"/var/lib/mercurial-server/.*",
"/var/lib/onak/.*",
"/var/lib/openvswitch/(pki/.*)?",
"/var/lib/vmm/(./.*)?", #682184
"/var/log/exim/.*",
"/var/log/exim4/.*",
"/var/spool/exim/.*",
"/var/spool/exim4/.*",
"/var/spool/news/.*",
"/var/spool/squid/(../.*)?",
"/var/www/.*",
# HACKS
"/lib/modules/.*/modules.*",
]
self.non_pedantic_ignore_patterns = [
"/tmp/.*"
]
settings = Settings()
on_panic_hooks = {}
counter = 0
def do_on_panic(hook):
global counter
cid = counter
counter += 1
on_panic_hooks[cid] = hook
return cid
def dont_do_on_panic(id):
del on_panic_hooks[id]
class TimeOffsetFormatter(logging.Formatter):
def __init__(self, fmt=None, datefmt=None):
self.startup_time = time.time()
logging.Formatter.__init__(self, fmt, datefmt)
def formatTime(self, record, datefmt):
t = time.time() - self.startup_time
t_min = int(t / 60)
t_sec = t % 60.0
return "%dm%.1fs" % (t_min, t_sec)
DUMP = logging.DEBUG - 1
HANDLERS = []
def setup_logging(log_level, log_file_name):
logging.addLevelName(DUMP, "DUMP")
logger = logging.getLogger()
logger.setLevel(log_level)
formatter = TimeOffsetFormatter("%(asctime)s %(levelname)s: %(message)s")
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(formatter)
logger.addHandler(handler)
HANDLERS.append(handler)
if log_file_name:
handler = logging.FileHandler(log_file_name)
handler.setFormatter(formatter)
logger.addHandler(handler)
HANDLERS.append(handler)
def dump(msg):
logger = logging.getLogger()
logger.log(DUMP, msg)
for handler in HANDLERS:
handler.flush()
def panic(exit=1):
for i in range(counter):
if i in on_panic_hooks:
on_panic_hooks[i]()
logging.error("piuparts run ends.")
sys.exit(exit)
def indent_string(str):
"""Indent all lines in a string with two spaces and return result."""
return "\n".join([" " + line for line in str.split("\n")])
def quote_spaces(vlist):
return ["'%s'" % x if ' ' in x else x for x in vlist]
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
def run(command, ignore_errors=False, timeout=0):
"""Run an external command and die with error message if it fails."""
def kill_subprocess(p, reason):
logging.error("Terminating command due to %s" % reason)
p.terminate()
for i in range(10):
time.sleep(0.5)
if p.poll() is not None:
break
else:
logging.error("Killing command due to %s" % reason)
p.kill()
p.wait()
assert type(command) == type([])
logging.debug("Starting command: %s" % command)
env = os.environ.copy()
env["LC_ALL"] = "C"
env["LANGUAGES"] = ""
env["PIUPARTS_OBJECTS"] = ' '.join(str(vobject) for vobject in settings.testobjects)
devnull = open('/dev/null', 'r')
p = subprocess.Popen(command, env=env, stdin=devnull,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = ""
excessive_output = False
if timeout > 0:
signal(SIGALRM, alarm_handler)
alarm(timeout)
try:
while p.poll() is None:
"""Read 64 KB chunks, but depending on the output buffering behavior
of the command we may get less even if more output is coming later.
Abort after reading max_command_output_size bytes."""
output += p.stdout.read(1 << 16)
if (len(output) > settings.max_command_output_size):
excessive_output = True
ignore_errors = False
alarm(0)
kill_subprocess(p, "excessive output")
output += "\n\n***** Command was terminated after exceeding output limit (%.2f MB) *****\n" \
% (settings.max_command_output_size / 1024. / 1024.)
break
if not excessive_output:
output += p.stdout.read(settings.max_command_output_size)
alarm(0)
except Alarm:
ignore_errors = False
kill_subprocess(p, "excessive runtime")
output += "\n\n***** Command was terminated after exceeding runtime limit (%s s) *****\n" % timeout
devnull.close()
if output:
dump("\n" + indent_string(output.rstrip("\n")))
if p.returncode == 0:
logging.debug("Command ok: %s" % repr(command))
elif ignore_errors:
logging.debug("Command failed (status=%d), but ignoring error: %s" %
(p.returncode, repr(command)))
else:
logging.error("Command failed (status=%d): %s\n%s" %
(p.returncode, repr(command), indent_string(output)))
panic()
return p.returncode, output
def create_temp_file():
"""Create a temporary file and return its full path."""
(fd, path) = tempfile.mkstemp(dir=settings.tmpdir)
logging.debug("Created temporary file %s" % path)
return (fd, path)
def create_file(name, contents):
"""Create a new file with the desired name and contents."""
try:
f = file(name, "w")
f.write(contents)
f.close()
except IOError, detail:
logging.error("Couldn't create file %s: %s" % (name, detail))
panic()
def remove_files(filenames):
"""Remove some files."""
for filename in filenames:
logging.debug("Removing %s" % filename)
try:
os.remove(filename)
except OSError, detail:
logging.error("Couldn't remove %s: %s" % (filename, detail))
panic()
def make_metapackage(name, depends, conflicts):
"""Return the path to a .deb created just for satisfying dependencies
Caller is responsible for removing the temporary directory containing the
.deb when finished.
"""
# Inspired by pbuilder's pbuilder-satisfydepends-aptitude
tmpdir = tempfile.mkdtemp(dir=settings.tmpdir)
panic_handler_id = do_on_panic(lambda: shutil.rmtree(tmpdir))
create_file(os.path.join(tmpdir, ".piuparts.tmpdir"), "metapackage creation")
old_umask = os.umask(0)
os.makedirs(os.path.join(tmpdir, name, 'DEBIAN'), mode = 0755)
os.umask(old_umask)
control = deb822.Deb822()
control['Package'] = name
control['Version'] = '0.invalid.0'
control['Architecture'] = 'all'
control['Maintainer'] = ('piuparts developers team '
'')
control['Description'] = ('Dummy package to satisfy dependencies - '
'created by piuparts\n'
' This package was created automatically by '
'piuparts and can safely be removed')
if depends:
control['Depends'] = depends
if conflicts:
control['Conflicts'] = conflicts
create_file(os.path.join(tmpdir, name, 'DEBIAN', 'control'),
control.dump())
run(['dpkg-deb', '-b', '--nocheck', os.path.join(tmpdir, name)])
dont_do_on_panic(panic_handler_id)
return os.path.join(tmpdir, name) + '.deb'
def split_path(pathname):
parts = []
while pathname:
(head, tail) = os.path.split(pathname)
#print "split '%s' => '%s' + '%s'" % (pathname, head, tail)
if tail:
parts.append(tail)
elif not head:
break
elif head == pathname:
parts.append(head)
break
pathname = head
return parts
def canonicalize_path(root, pathname):
"""Canonicalize a path name, simulating chroot at 'root'.
When resolving the symlink, pretend (similar to chroot) that
'root' is the root of the filesystem. Also resolve '..' and
'.' components. This should not escape the chroot below
'root', but for security concerns, use chroot and have the
kernel resolve symlinks instead.
"""
#print "\nCANONICALIZE %s %s" % (root, pathname)
seen = []
parts = split_path(pathname)
#print "PARTS ", list(reversed(parts))
path = "/"
while parts:
tag = "\n".join(parts + [path])
#print "TEST '%s' + " % path, list(reversed(parts))
if tag in seen or len(seen) > 1024:
fullpath = os.path.join(path, *reversed(parts))
#print "LOOP %s" % fullpath
path = fullpath
logging.error("ELOOP: Too many symbolic links in '%s'" % path)
break
seen.append(tag)
part = parts.pop()
# Using normpath() to cleanup '.', '..' and multiple slashes.
# Removing a suffix 'foo/..' is safe here since it can't change the
# meaning of 'path' because it contains no symlinks - they have been
# resolved already.
newpath = os.path.normpath(os.path.join(path, part))
rootedpath = os.path.join(root, newpath[1:])
if newpath == "/":
path = "/"
elif os.path.islink(rootedpath):
target = os.readlink(rootedpath)
#print "LINK to '%s'" % target
if os.path.isabs(target):
path = "/"
parts.extend(split_path(target))
else:
path = newpath
#print "FINAL '%s'" % path
return path
def is_broken_symlink(root, dirpath, filename):
"""Is symlink dirpath+filename broken?"""
if dirpath[:len(root)] == root:
dirpath = dirpath[len(root):]
pathname = canonicalize_path(root, os.path.join(dirpath, filename))
pathname = os.path.join(root, pathname[1:])
# The symlink chain, if any, has now been resolved. Does the target
# exist?
#print "EXISTS ", pathname, os.path.exists(pathname)
return not os.path.exists(pathname)
class IsBrokenSymlinkTests(unittest.TestCase):
testdir = "is-broken-symlink-testdir"
def symlink(self, target, name):
pathname = os.path.join(self.testdir, name)
os.symlink(target, pathname)
self.symlinks.append(pathname)
def setUp(self):
self.symlinks = []
os.mkdir(self.testdir)
self.symlink("notexist", "relative-broken")
self.symlink("relative-broken", "relative-broken-to-symlink")
self.symlink(".", "relative-works")
self.symlink("relative-works", "relative-works-to-symlink")
self.symlink("/etc", "absolute-broken")
self.symlink("absolute-broken", "absolute-broken-to-symlink")
self.symlink("/", "absolute-works")
self.symlink("/absolute-works", "absolute-works-to-symlink")
os.mkdir(os.path.join(self.testdir, "dir"))
self.symlink("dir", "dir-link")
os.mkdir(os.path.join(self.testdir, "dir/subdir"))
self.symlink("subdir", "dir/subdir-link")
self.symlink("notexist/", "trailing-slash-broken")
self.symlink("dir/", "trailing-slash-works")
self.symlink("selfloop", "selfloop")
self.symlink("/absolute-selfloop", "absolute-selfloop")
self.symlink("../dir/selfloop", "dir/selfloop")
self.symlink("../dir-link/selfloop", "dir/selfloop1")
self.symlink("../../dir/subdir/selfloop", "dir/subdir/selfloop")
self.symlink("../../dir-link/subdir/selfloop", "dir/subdir/selfloop1")
self.symlink("../../link/subdir-link/selfloop", "dir/subdir/selfloop2")
self.symlink("../../dir-link/subdir-link/selfloop", "dir/subdir/selfloop3")
self.symlink("explode/bomb", "explode")
def tearDown(self):
shutil.rmtree(self.testdir)
def testRelativeBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"relative-broken"))
def testRelativeBrokenToSymlink(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"relative-broken-to-symlink"))
def testAbsoluteBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"absolute-broken"))
def testAbsoluteBrokenToSymlink(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"absolute-broken-to-symlink"))
def testTrailingSlashBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"trailing-slash-broken"))
def testSelfLoopBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"selfloop"))
def testExpandingSelfLoopBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"explode"))
def testAbsoluteSelfLoopBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"absolute-selfloop"))
def testSubdirSelfLoopBroken(self):
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"dir/selfloop"))
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"dir/selfloop1"))
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"dir/subdir/selfloop"))
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"dir/subdir/selfloop1"))
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"dir/subdir/selfloop2"))
self.failUnless(is_broken_symlink(self.testdir, self.testdir,
"dir/subdir/selfloop3"))
def testRelativeWorks(self):
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"relative-works"))
def testRelativeWorksToSymlink(self):
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"relative-works-to-symlink"))
def testAbsoluteWorks(self):
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"absolute-works"))
def testAbsoluteWorksToSymlink(self):
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"absolute-works-to-symlink"))
def testTrailingSlashWorks(self):
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"trailing-slash-works"))
def testMultiLevelNestedSymlinks(self):
# target/first-link -> ../target/second-link -> ../target
os.mkdir(os.path.join(self.testdir, "target"))
self.symlink("../target", "target/second-link")
self.symlink("../target/second-link", "target/first-link")
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"target/first-link"))
def testMultiLevelNestedAbsoluteSymlinks(self):
# first-link -> /second-link/final-target
# second-link -> /target-dir
os.mkdir(os.path.join(self.testdir, "final-dir"))
os.mkdir(os.path.join(self.testdir, "final-dir/final-target"))
self.symlink("/second-link/final-target", "first-link")
self.symlink("/final-dir", "second-link")
self.failIf(is_broken_symlink(self.testdir, self.testdir,
"first-link"))
class Chroot:
"""A chroot for testing things in."""
def __init__(self):
self.name = None
self.bootstrapped = False
def create_temp_dir(self):
"""Create a temporary directory for the chroot."""
self.name = tempfile.mkdtemp(dir=settings.tmpdir)
create_file(os.path.join(self.name, ".piuparts.tmpdir"), "chroot")
os.chmod(self.name, 0755)
logging.debug("Created temporary directory %s" % self.name)
def create(self, temp_tgz = None):
"""Create a chroot according to user's wishes."""
self.panic_handler_id = do_on_panic(self.remove)
if not settings.schroot:
self.create_temp_dir()
if temp_tgz:
self.unpack_from_tgz(temp_tgz)
elif settings.basetgz:
self.unpack_from_tgz(settings.basetgz)
elif settings.lvm_volume:
self.setup_from_lvm(settings.lvm_volume)
elif settings.existing_chroot:
self.setup_from_dir(settings.existing_chroot)
elif settings.schroot:
self.setup_from_schroot(settings.schroot)
else:
self.setup_minimal_chroot()
if not settings.schroot:
self.mount_proc()
self.mount_selinux()
self.configure_chroot()
if settings.basetgz or settings.schroot:
self.run(["apt-get", "-yf", "dist-upgrade"])
self.minimize()
# Copy scripts dirs into the chroot, merging all dirs together,
# later files overwriting earlier ones.
if settings.scriptsdirs:
dest = self.relative("tmp/scripts/")
if not os.path.exists(self.relative("tmp/scripts/")):
os.mkdir(dest)
for sdir in settings.scriptsdirs:
logging.debug("Copying scriptsdir %s to %s" % (sdir, dest))
for sfile in os.listdir(sdir):
if (sfile.startswith("post_") or sfile.startswith("pre_")) \
and not ".dpkg-" in sfile \
and os.path.isfile(os.path.join(sdir, sfile)):
shutil.copy(os.path.join(sdir, sfile), dest)
# Run custom scripts after creating the chroot.
self.run_scripts("post_setup")
if settings.savetgz and not temp_tgz:
self.pack_into_tgz(settings.savetgz)
def remove(self):
"""Remove a chroot and all its contents."""
if not settings.keep_tmpdir and os.path.exists(self.name):
self.terminate_running_processes()
if not settings.schroot:
self.unmount_selinux()
self.unmount_proc()
if settings.lvm_volume:
logging.debug('Unmounting and removing LVM snapshot %s' % self.lvm_snapshot_name)
run(['umount', self.name])
run(['lvremove', '-f', self.lvm_snapshot])
if settings.schroot:
logging.debug("Terminate schroot session '%s'" % self.name)
run(['schroot', '--end-session', '--chroot', "session:" + self.schroot_session])
if not settings.schroot:
run(['rm', '-rf', '--one-file-system', self.name])
if os.path.exists(self.name):
create_file(os.path.join(self.name, ".piuparts.tmpdir"), "removal failed")
logging.debug("Removed directory tree at %s" % self.name)
elif settings.keep_tmpdir:
if settings.schroot:
logging.debug("Keeping schroot session %s at %s" % (self.schroot_session, self.name))
else:
logging.debug("Keeping directory tree at %s" % self.name)
dont_do_on_panic(self.panic_handler_id)
def was_bootstrapped(self):
return self.bootstrapped
def create_temp_tgz_file(self):
"""Return the path to a file to be used as a temporary tgz file"""
# Yes, create_temp_file() would work just as well, but putting it in
# the interface for Chroot allows the VirtServ hack to work.
(fd, temp_tgz) = create_temp_file()
os.close(fd)
return temp_tgz
def remove_temp_tgz_file(self, temp_tgz):
"""Remove the file that was used as a temporary tgz file"""
# Yes, remove_files() would work just as well, but putting it in
# the interface for Chroot allows the VirtServ hack to work.
remove_files([temp_tgz])
def pack_into_tgz(self, result):
"""Tar and compress all files in the chroot."""
self.run(["apt-get", "clean"])
logging.debug("Saving %s to %s." % (self.name, result))
(fd, tmpfile) = tempfile.mkstemp(dir=os.path.dirname(result))
os.close(fd)
cleanup_tmpfile = lambda: os.remove(tmpfile)
panic_handler_id = do_on_panic(cleanup_tmpfile)
run(['tar', '-czf', tmpfile, '--one-file-system', '--exclude', 'tmp/scripts', '-C', self.name, './'])
os.chmod(tmpfile, 0644)
os.rename(tmpfile, result)
dont_do_on_panic(panic_handler_id)
def unpack_from_tgz(self, tarball):
"""Unpack a tarball to a chroot."""
logging.debug("Unpacking %s into %s" % (tarball, self.name))
prefix = []
if settings.eatmydata and os.path.isfile('/usr/bin/eatmydata'):
prefix.append('eatmydata')
run(prefix + ["tar", "-C", self.name, "-zxf", tarball])
def setup_from_schroot(self, schroot):
self.schroot_session = schroot.split(":")[1] + "-" + str(uuid.uuid1()) + "-piuparts"
run(['schroot', '--begin-session', '--chroot', schroot , '--session-name', self.schroot_session])
ret_code, output = run(['schroot', '--chroot', "session:" + self.schroot_session, '--location'])
self.name = output.strip()
logging.info("New schroot session in '%s'" % self.name);
def setup_from_lvm(self, lvm_volume):
"""Create a chroot by creating an LVM snapshot."""
self.lvm_base = os.path.dirname(lvm_volume)
self.lvm_vol_name = os.path.basename(lvm_volume)
self.lvm_snapshot_name = self.lvm_vol_name + "-" + str(uuid.uuid1());
self.lvm_snapshot = os.path.join(self.lvm_base, self.lvm_snapshot_name)
logging.debug("Creating LVM snapshot %s from %s" % (self.lvm_snapshot, lvm_volume))
run(['lvcreate', '-n', self.lvm_snapshot, '-s', lvm_volume, '-L', settings.lvm_snapshot_size])
logging.info("Mounting LVM snapshot to %s" % self.name);
run(['mount', self.lvm_snapshot, self.name])
def setup_from_dir(self, dirname):
"""Create chroot from an existing one."""
logging.debug("Copying %s into %s" % (dirname, self.name))
for name in os.listdir(dirname):
src = os.path.join(dirname, name)
dst = os.path.join(self.name, name)
run(["cp", "-ax", src, dst])
def run(self, command, ignore_errors=False):
prefix = []
if settings.eatmydata and os.path.isfile(os.path.join(self.name,
'usr/bin/eatmydata')):
prefix.append('eatmydata')
if settings.schroot:
return run(["schroot", "--preserve-environment", "--run-session", "--chroot", "session:" + self.schroot_session, "--directory", "/", "-u", "root", "--"] + prefix + command,
ignore_errors=ignore_errors, timeout=settings.max_command_runtime)
else:
return run(["chroot", self.name] + prefix + command,
ignore_errors=ignore_errors, timeout=settings.max_command_runtime)
def create_apt_sources(self, distro):
"""Create an /etc/apt/sources.list with a given distro."""
lines = []
lines.extend(settings.distro_config.get_deb_lines(
distro, settings.debian_mirrors[0][1]))
for mirror, components in settings.debian_mirrors[1:]:
lines.append("deb %s %s %s" %
(mirror, distro, " ".join(components)))
for repo in settings.extra_repos:
lines.append(repo)
create_file(self.relative("etc/apt/sources.list"),
"\n".join(lines) + "\n")
logging.debug("sources.list:\n" + indent_string("\n".join(lines)))
def enable_testdebs_repo(self, update=True):
if settings.testdebs_repo:
if settings.testdebs_repo.startswith("deb"):
debline = settings.testdebs_repo
elif settings.testdebs_repo.startswith("/"):
debline = "deb file://%s ./" % settings.testdebs_repo
else:
debline = "deb %s ./" % settings.testdebs_repo
logging.debug("enabling testdebs repository '%s'" % debline)
create_file(self.relative("etc/apt/sources.list.d/piuparts-testdebs-repo.list"), debline + "\n")
if update:
self.run(["apt-get", "update"])
def disable_testdebs_repo(self):
if settings.testdebs_repo:
logging.debug("disabling testdebs repository")
remove_files([self.relative("etc/apt/sources.list.d/piuparts-testdebs-repo.list")])
def create_apt_conf(self):
"""Create /etc/apt/apt.conf.d/piuparts inside the chroot."""
lines = ['APT::Get::Assume-Yes "yes";\n']
lines.append('APT::Install-Recommends "%d";\n' % int(settings.install_recommends))
lines.append('APT::Install-Suggests "0";\n')
lines.append('APT::Get::AllowUnauthenticated "%s";\n' % settings.apt_unauthenticated)
lines.append('Acquire::PDiffs "false";\n')
if settings.proxy:
proxy = settings.proxy
elif "http_proxy" in os.environ:
proxy = os.environ["http_proxy"]
else:
proxy = None;
pat = re.compile(r"^Acquire::http::Proxy\s+\"([^\"]+)\"", re.I);
p = subprocess.Popen(["apt-config", "dump"],
stdout=subprocess.PIPE)
stdout, _ = p.communicate()
if stdout:
for line in stdout.split("\n"):
m = re.match(pat, line)
if proxy is None and m:
proxy = m.group(1)
if proxy:
lines.append('Acquire::http::Proxy "%s";\n' % proxy)
if settings.dpkg_force_unsafe_io:
lines.append('Dpkg::Options {"--force-unsafe-io";};\n')
if settings.dpkg_force_confdef:
lines.append('Dpkg::Options {"--force-confdef";};\n')
create_file(self.relative("etc/apt/apt.conf.d/piuparts"),
"".join(lines))
def create_dpkg_conf(self):
"""Create /etc/dpkg/dpkg.cfg.d/piuparts inside the chroot."""
lines = []
if settings.dpkg_force_unsafe_io:
lines.append('force-unsafe-io\n')
if settings.dpkg_force_confdef:
lines.append('force-confdef\n')
logging.info("Warning: dpkg has been configured to use the force-confdef option. This will hide problems, see #466118.")
if lines:
if not os.path.exists(self.relative("etc/dpkg/dpkg.cfg.d")):
os.mkdir(self.relative("etc/dpkg/dpkg.cfg.d"))
create_file(self.relative("etc/dpkg/dpkg.cfg.d/piuparts"),
"".join(lines))
def create_policy_rc_d(self):
"""Create a policy-rc.d that prevents daemons from running."""
full_name = self.relative("usr/sbin/policy-rc.d")
policy = "#!/bin/sh\n"
if settings.allow_database:
policy += 'test "$1" = "mysql" && exit 0\n'
policy += 'test "$1" = "postgresql" && exit 0\n'
policy += 'test "$1" = "postgresql-8.3" && exit 0\n'
policy += 'test "$1" = "firebird2.5-super" && exit 0\n'
policy += "exit 101\n"
create_file(full_name, policy)
os.chmod(full_name, 0755)
logging.debug("Created policy-rc.d and chmodded it.")
def setup_minimal_chroot(self):
"""Set up a minimal Debian system in a chroot."""
logging.debug("Setting up minimal chroot for %s at %s." %
(settings.debian_distros[0], self.name))
prefix = []
if settings.eatmydata and os.path.isfile('/usr/bin/eatmydata'):
prefix.append('eatmydata')
if settings.do_not_verify_signatures:
logging.info("Warning: not using --keyring option when running debootstrap!")
options = [settings.keyringoption]
if settings.eatmydata:
options.append('--include=eatmydata')
options.append('--components=%s' % ','.join(settings.debian_mirrors[0][1]))
if settings.arch:
options.append('--arch=%s' % settings.arch)
run(prefix + ["debootstrap", "--variant=minbase"] + options +
[settings.debian_distros[0], self.name, settings.distro_config.get_mirror(settings.debian_distros[0])])
self.bootstrapped = True
def minimize(self):
"""Minimize a chroot by removing (almost all) unnecessary packages"""
if settings.skip_minimize or not settings.minimize:
return
self.run(["apt-get", "install", "debfoster"])
debfoster_command = ["debfoster"] + settings.debfoster_options
if settings.eatmydata:
debfoster_command.append("eatmydata")
self.run(debfoster_command)
remove_files([self.relative("var/lib/debfoster/keepers")])
self.run(["dpkg", "--purge", "debfoster"])
def configure_chroot(self):
"""Configure a chroot according to current settings"""
os.environ["PIUPARTS_DISTRIBUTION"] = settings.debian_distros[0]
if not settings.keep_sources_list:
self.create_apt_sources(settings.debian_distros[0])
self.create_apt_conf()
self.create_dpkg_conf()
self.create_policy_rc_d()
for bindmount in settings.bindmounts:
run(["mkdir", "-p", self.relative(bindmount)])
run(["mount", "-obind", bindmount, self.relative(bindmount)])
self.run(["apt-get", "update"])
def upgrade_to_distros(self, distros, packages):
"""Upgrade a chroot installation to each successive distro."""
for distro in distros:
logging.debug("Upgrading %s to %s" % (self.name, distro))
os.environ["PIUPARTS_DISTRIBUTION_NEXT"] = distro
self.create_apt_sources(distro)
# Run custom scripts before upgrade
self.run_scripts("pre_distupgrade")
self.run(["apt-get", "update"])
self.run(["apt-get", "-yf", "dist-upgrade"])
os.environ["PIUPARTS_DISTRIBUTION_PREV"] = os.environ["PIUPARTS_DISTRIBUTION"]
os.environ["PIUPARTS_DISTRIBUTION"] = distro
# Sometimes dist-upgrade won't upgrade the packages we want
# to test because the new version depends on a newer library,
# and installing that would require removing the old version
# of the library, and we've told apt-get not to remove
# packages. So, we force the installation like this.
if packages:
known_packages = self.get_known_packages(packages + settings.extra_old_packages)
self.install_packages_by_name(known_packages)
# Run custom scripts after upgrade
self.run_scripts("post_distupgrade")
self.check_for_no_processes()
def get_known_packages(self, packages):
"""Does apt-get (or apt-cache) know about a set of packages?"""
known_packages = []
new_packages = []
for name in packages:
(status, output) = self.run(["apt-cache", "show", name],
ignore_errors=True)
# apt-cache reports status for some virtual packages and packages
# in status config-files-remaining state without installation
# candidate -- but only real packages have Filename/MD5sum/SHA*
if status != 0 or re.search(r'^(Filename|MD5sum|SHA1|SHA256):', output, re.M) is None:
new_packages.append(name)
else:
known_packages.append(name)
if not known_packages:
logging.info("apt-cache does not know about any of the requested packages")
else:
logging.info("apt-cache knows about the following packages: " +
", ".join(known_packages))
if new_packages:
logging.info("the following packages are not in the archive: " +
", ".join(new_packages))
return known_packages
def copy_files(self, source_names, target_name):
"""Copy files in 'source_name' to file/dir 'target_name', relative
to the root of the chroot."""
target_name = self.relative(target_name)
logging.debug("Copying %s to %s" %
(", ".join(source_names), target_name))
for source_name in source_names:
try:
shutil.copy(source_name, target_name)
except IOError, detail:
logging.error("Error copying %s to %s: %s" %
(source_name, target_name, detail))
panic()
def list_installed_files (self, pre_info, post_info):
"""List the new files installed, removed and modified between two dir trees.
Actually, it is a nice output of the funcion diff_meta_dat."""
(new, removed, modified) = diff_meta_data(pre_info, post_info)
file_owners = self.get_files_owned_by_packages()
if new:
logging.debug("New installed files on system:\n" + file_list(new, file_owners))
else:
logging.debug("The package did not install any new file.\n")
if removed:
logging.debug("The following files have disappeared:\n" +
file_list(removed, file_owners))
if modified:
logging.debug("The following files have been modified:\n" +
file_list(modified, file_owners))
else:
logging.debug("The package did not modify any file.\n")
def install_packages(self, package_files, packages, with_scripts=True):
if package_files:
self.install_package_files(package_files, packages, with_scripts=with_scripts)
else:
self.install_packages_by_name(packages, with_scripts=with_scripts)
def install_package_files(self, package_files, packages=None, with_scripts=False):
if packages and settings.testdebs_repo:
self.install_packages_by_name(packages, with_scripts=with_scripts)
return
if package_files:
self.copy_files(package_files, "tmp")
tmp_files = [os.path.basename(a) for a in package_files]
tmp_files = [os.path.join("tmp", name) for name in tmp_files]
if with_scripts:
self.run_scripts("pre_install")
apt_get_install = ["apt-get", "-yf"]
apt_get_install.extend(settings.distro_config.get_target_flags(
os.environ["PIUPARTS_DISTRIBUTION"]))
apt_get_install.append("install")
if settings.list_installed_files:
pre_info = self.save_meta_data()
self.run(["dpkg", "-i"] + tmp_files, ignore_errors=True)
self.list_installed_files (pre_info, self.save_meta_data())
self.run(apt_get_install)
self.list_installed_files (pre_info, self.save_meta_data())
else:
self.run(["dpkg", "-i"] + tmp_files, ignore_errors=True)
self.run(apt_get_install)
logging.info ("Installation of %s ok", tmp_files)
if with_scripts:
self.run_scripts("post_install")
remove_files([self.relative(name) for name in tmp_files])
def install_packages_by_name(self, packages, with_scripts=True):
if packages:
if with_scripts:
self.run_scripts("pre_install")
self.run(["apt-cache", "policy"])
self.run(["apt-cache", "policy"] + [p.split("=", 1)[0].strip() for p in packages])
apt_get_install = ["apt-get", "-y"]
apt_get_install.extend(settings.distro_config.get_target_flags(
os.environ["PIUPARTS_DISTRIBUTION"]))
apt_get_install.append("install")
apt_get_install.extend(packages)
if settings.list_installed_files:
pre_info = self.save_meta_data()
self.run(apt_get_install)
self.list_installed_files (pre_info, self.save_meta_data())
else:
self.run(apt_get_install)
if with_scripts:
self.run_scripts("post_install")
def get_selections(self):
"""Get current package selections in a chroot."""
(status, output) = self.run(["dpkg", "--get-selections", "*"])
vlist = [line.split() for line in output.split("\n") if line.strip()]
vdict = {}
for name, status in vlist:
vdict[name] = status
return vdict
def get_diversions(self):
"""Get current dpkg-divert --list in a chroot."""
if not settings.check_broken_diversions:
return
(status, output) = self.run(["dpkg-divert", "--list"])
return output.split("\n")
def get_modified_diversions(self, pre_install_diversions, post_install_diversions = None):
"""Check that diversions in chroot are identical (though potentially reordered)."""
if post_install_diversions is None:
post_install_diversions = self.get_diversions()
removed = [ln for ln in pre_install_diversions if not ln in post_install_diversions]
added = [ln for ln in post_install_diversions if not ln in pre_install_diversions]
return (removed, added)
def check_debsums(self):
(status, output) = run(["debsums", "--root", self.name, "-ac"], ignore_errors=True)
if status != 0:
logging.error("FAIL: debsums reports modifications inside the chroot:\n%s" %
indent_string(output.replace(self.name, "")))
if not settings.warn_on_debsums_errors:
panic()
def check_adequate(self, packages):
"""Run adequate and categorize output according to our needs. """
packages = [p.split("=", 1)[0].strip() for p in packages if not p.endswith("=None")]
if packages and settings.adequate and os.path.isfile('/usr/bin/adequate'):
adequate_tags = [
'bin-or-sbin-binary-requires-usr-lib-library',
'library-not-found',
'missing-copyright-file',
'py-file-not-bytecompiled',
'pyshared-file-not-bytecompiled',
'undefined-symbol',
'missing-symbol-version-information',
'symbol-size-mismatch',
'incompatible-licenses',
'ldd',
]
boring_tags = [
'obsolete-conffile',
'broken-symlink',
]
ignored_tags = [ ]
(status, output) = run(["adequate", "--root", self.name] + packages, ignore_errors=True)
for tag in ignored_tags:
# ignore some tags
_regex = '^[^:]+: '+tag+' .*\n'
output = re.compile(_regex, re.MULTILINE).sub('', output)
if output:
inadequate_results = ''
boring_results = ''
for tag in adequate_tags:
if ' '+tag+' ' in output:
inadequate_results += ' '+tag+' '
for tag in boring_tags:
if ' '+tag+' ' in output:
boring_results += ' '+tag+' '
if settings.warn_if_inadequate:
error_code = 'WARN'
else:
error_code = 'FAIL'
logging.error("%s: Inadequate results from running adequate!\n%s" %
(error_code, indent_string(output.replace(self.name, ""))))
if inadequate_results:
logging.error("%s: Running adequate resulted in inadequate tags found: %s" % (error_code, inadequate_results))
if boring_results:
logging.error("%s: Running adequate resulted in less interesting tags found: %s" % (error_code, boring_results))
if not boring_results and not inadequate_results:
logging.error("%s: Found unknown tags running adequate." % error_code)
if status != 0:
logging.error("%s: Exit code from adequate was %s!" % (error_code,status))
if not settings.warn_if_inadequate:
panic()
def list_paths_with_symlinks(self):
file_owners = self.get_files_owned_by_packages()
bad = []
overwrites = False
for f in sorted(file_owners.keys()):
dn, fn = os.path.split(f)
dc = canonicalize_path(self.name, dn)
if dn != dc:
fc = os.path.join(dc, fn)
of = ", ".join(file_owners[f])
if fc in file_owners:
overwrites = True
ofc = ", ".join(file_owners[fc])
else:
ofc = "?"
bad.append("%s (%s) != %s (%s)" %(f, of, fc, ofc))
if bad:
if overwrites:
logging.error("FAIL: silently overwrites files via directory symlinks:\n" +
indent_string("\n".join(bad)))
else:
logging.info("dirname part contains a symlink:\n" +
indent_string("\n".join(bad)))
def remove_packages(self, packages):
"""Remove packages in a chroot."""
if packages:
packages = [p.split("=", 1)[0].strip() for p in packages]
self.run(["apt-get", "remove"] + packages, ignore_errors=True)
def purge_packages(self, packages):
"""Purge packages in a chroot."""
if packages:
packages = [p.split("=", 1)[0].strip() for p in packages]
self.run(["dpkg", "--purge"] + packages, ignore_errors=True)
def restore_selections(self, selections, packages_qualified):
"""Restore package selections in a chroot to the state in
'selections'."""
packages = [p.split("=", 1)[0].strip() for p in packages_qualified]
changes = diff_selections(self, selections)
deps = {}
nondeps = {}
for name, state in changes.iteritems():
if name in packages:
nondeps[name] = state
else:
deps[name] = state
deps_to_remove = [name for name, state in deps.iteritems()
if state == "remove"]
deps_to_purge = [name for name, state in deps.iteritems()
if state == "purge"]
nondeps_to_remove = [name for name, state in nondeps.iteritems()
if state == "remove"]
nondeps_to_purge = [name for name, state in nondeps.iteritems()
if state == "purge"]
deps_to_install = [name for name, state in deps.iteritems()
if state == "install"]
self.list_paths_with_symlinks()
self.check_debsums()
self.check_adequate(packages_qualified)
# Run custom scripts before removing all packages.
self.run_scripts("pre_remove")
# First remove all packages (and reinstall missing ones).
self.remove_packages(deps_to_remove)
self.remove_packages(deps_to_remove + deps_to_purge +
nondeps_to_remove + nondeps_to_purge +
["%s+" % x for x in deps_to_install])
# Run custom scripts after removing all packages.
self.run_scripts("post_remove")
if not settings.skip_cronfiles_test:
cronfiles, cronfiles_list = self.check_if_cronfiles(packages)
if not settings.skip_cronfiles_test and cronfiles:
self.check_output_cronfiles(cronfiles_list)
if not settings.skip_logrotatefiles_test:
logrotatefiles, logrotatefiles_list = self.check_if_logrotatefiles(packages)
if not settings.skip_logrotatefiles_test and logrotatefiles:
installed = self.install_logrotate()
self.check_output_logrotatefiles(logrotatefiles_list)
self.purge_packages(installed)
# Then purge all packages being depended on.
self.purge_packages(deps_to_purge)
# Finally, purge actual packages.
self.purge_packages(nondeps_to_purge)
# Run custom scripts after purge all packages.
self.run_scripts("post_purge")
# Now do a final run to see that everything worked.
self.run(["dpkg", "--purge", "--pending"])
self.run(["dpkg", "--remove", "--pending"])
def save_meta_data(self):
"""Return the filesystem meta data for all objects in the chroot."""
self.run(["apt-get", "clean"])
root = self.relative(".")
vdict = {}
proc = os.path.join(root, "proc")
for dirpath, dirnames, filenames in os.walk(root):
assert dirpath[:len(root)] == root
if dirpath[:len(proc) + 1] in [proc, proc + "/"]:
continue
for name in [dirpath] + \
[os.path.join(dirpath, f) for f in filenames]:
st = os.lstat(name)
if stat.S_ISLNK(st.st_mode):
target = os.readlink(name)
else:
target = None
if stat.S_ISDIR(st.st_mode):
name += "/"
vdict[name[len(root):]] = (st, target)
return vdict
def relative(self, pathname):
if pathname.startswith('/'):
return os.path.join(self.name, pathname[1:])
return os.path.join(self.name, pathname)
def get_files_owned_by_packages(self):
"""Return dict[filename] = [packagenamelist]."""
vdir = self.relative("var/lib/dpkg/info")
vdict = {}
for basename in os.listdir(vdir):
if basename.endswith(".list"):
pkg = basename[:-len(".list")]
f = file(os.path.join(vdir, basename), "r")
for line in f:
pathname = line.strip()
if pathname in vdict:
vdict[pathname].append(pkg)
else:
vdict[pathname] = [pkg]
f.close()
return vdict
def check_for_no_processes(self, fail=None):
"""Check there are no processes running inside the chroot."""
(status, output) = run(["lsof", "-w", "+D", self.name], ignore_errors=True)
count = len(output.split("\n")) - 1
if count > 0:
if fail is None:
fail = not settings.allow_database
logging.error("%s: Processes are running inside chroot:\n%s" %
("FAIL" if fail else "WARN", indent_string(output)))
if fail:
self.terminate_running_processes()
panic()
def terminate_running_processes(self):
"""Terminate all processes running in the chroot."""
seen = []
while True:
p = subprocess.Popen(["lsof", "-t", "+D", self.name],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
stdout, _ = p.communicate()
if not stdout:
break
pidlist = reversed([int(pidstr) for pidstr in stdout.split("\n") if len(pidstr) and int(pidstr) > 0])
if not pidlist:
break
for pid in pidlist:
try:
signo = (SIGTERM, SIGKILL)[pid in seen]
os.kill(pid, signo)
seen.append(pid)
logging.debug("kill -%d %d" % (signo, pid))
time.sleep(0.25)
except OSError:
pass
time.sleep(5)
def mount_selinux(self):
if selinux_enabled():
run(["mkdir", "-p", self.relative("/selinux")])
run(["mount", "-t", "selinuxfs", "/selinux", self.relative("/selinux")])
logging.info("SElinux mounted into chroot")
def unmount_selinux(self):
if selinux_enabled():
run(["umount", self.relative("/selinux")])
logging.info("SElinux unmounted from chroot")
def mount_proc(self):
"""Mount /proc inside chroot."""
self.run(["mount", "-t", "proc", "proc", "/proc"])
def unmount_proc(self):
"""Unmount /proc inside chroot."""
self.run(["umount", "/proc"], ignore_errors=True)
for bindmount in settings.bindmounts:
run(["umount", self.relative(bindmount)], ignore_errors=True)
def is_ignored(self, pathname):
"""Is a file (or dir or whatever) to be ignored?"""
if pathname in settings.ignored_files:
return True
for pattern in settings.ignored_patterns:
if re.search('^' + pattern + '$', pathname):
return True
return False
def check_for_broken_symlinks(self):
"""Check that all symlinks in chroot are non-broken."""
if not settings.check_broken_symlinks:
return
broken = []
for dirpath, dirnames, filenames in os.walk(self.name):
# Remove /proc within chroot to avoid lots of spurious errors.
if dirpath == self.name and "proc" in dirnames:
dirnames.remove("proc")
for filename in filenames:
full_name = name = os.path.join(dirpath, filename)
if name.startswith(self.name):
name = name[len(self.name):]
ret = is_broken_symlink(self.name, dirpath, filename)
if ret and not self.is_ignored(name):
try:
target = os.readlink(full_name)
except os.error:
target = ""
broken.append("%s -> %s" % (name, target))
if broken:
if settings.warn_broken_symlinks:
logging.error("WARN: Broken symlinks:\n%s" %
indent_string("\n".join(broken)))
else:
logging.error("FAIL: Broken symlinks:\n%s" %
indent_string("\n".join(broken)))
panic()
else:
logging.debug("No broken symlinks as far as we can find.")
def check_if_cronfiles(self, packages):
"""Check if the packages have cron files under /etc/cron.d and in case positive,
it returns the list of files. """
vdir = self.relative("var/lib/dpkg/info")
vlist = []
has_cronfiles = False
for p in packages:
basename = p + ".list"
if not os.path.exists(os.path.join(vdir, basename)):
continue
f = file(os.path.join(vdir, basename), "r")
for line in f:
pathname = line.strip()
if pathname.startswith("/etc/cron."):
if os.path.isfile(self.relative(pathname.strip("/"))):
st = os.lstat(self.relative(pathname.strip("/")))
mode = st[stat.ST_MODE]
# XXX /etc/cron.d/ files are NOT executables
if (mode & stat.S_IEXEC):
if not has_cronfiles:
has_cronfiles = True
vlist.append(pathname)
logging.info("Package " + p + " contains cron file: " + pathname)
f.close()
return has_cronfiles, vlist
def check_output_cronfiles (self, list):
"""Check if a given list of cronfiles has any output. Executes
cron file as cron would do (except for SHELL)"""
failed = False
for vfile in list:
if not os.path.exists(self.relative(vfile.strip("/"))):
continue
(retval, output) = self.run([vfile])
if output:
failed = True
logging.error("FAIL: Cron file %s has output with package removed" % vfile)
if failed:
panic()
def check_if_logrotatefiles(self, packages):
"""Check if the packages have logrotate files under /etc/logrotate.d and in case positive,
it returns the list of files. """
vdir = self.relative("var/lib/dpkg/info")
vlist = []
has_logrotatefiles = False
for p in packages:
basename = p + ".list"
if not os.path.exists(os.path.join(vdir, basename)):
continue
f = file(os.path.join(vdir, basename), "r")
for line in f:
pathname = line.strip()
if pathname.startswith("/etc/logrotate.d/"):
if os.path.isfile(self.relative(pathname.strip("/"))):
if not has_logrotatefiles:
has_logrotatefiles = True
vlist.append(pathname)
logging.info("Package " + p + " contains logrotate file: " + pathname)
f.close()
return has_logrotatefiles, vlist
def install_logrotate(self):
"""Install logrotate for check_output_logrotatefiles, and return the
list of packages that were installed"""
old_selections = self.get_selections()
self.run(['apt-get', 'install', '-y', 'logrotate'])
diff = diff_selections(self, old_selections)
return diff.keys()
def check_output_logrotatefiles (self, list):
"""Check if a given list of logrotatefiles has any output. Executes
logrotate file as logrotate would do from cron (except for SHELL)"""
failed = False
for vfile in list:
if not os.path.exists(self.relative(vfile.strip("/"))):
continue
(retval, output) = self.run(['/usr/sbin/logrotate', vfile])
if output or retval != 0:
failed = True
logging.error("FAIL: Logrotate file %s exits with error or has output with package removed" % file)
if failed:
panic()
def run_scripts (self, step):
""" Run custom scripts to given step post-install|remove|purge"""
if not settings.scriptsdirs:
return
logging.info("Running scripts "+ step)
basepath = self.relative("tmp/scripts/")
if not os.path.exists(basepath):
logging.error("Scripts directory %s does not exist" % basepath)
panic()
list_scripts = os.listdir(basepath)
list_scripts.sort()
for vfile in list_scripts:
if vfile.startswith(step):
script = os.path.join("tmp/scripts", vfile)
self.run([script])
class VirtServ(Chroot):
# Provides a thing that looks to the rest of piuparts much like
# a chroot but is actually provided by an adt virtualisation server.
# See /usr/share/doc/autopkgtest/README.virtualisation-server.
def __init__(self, cmdline):
self._cmdline = cmdline
self.name = '/ADT-VIRT'
self._vs = None
def _awaitok(self, cmd):
r = self._vs.stdout.readline().rstrip('\n')
l = r.split(' ')
if l[0] != 'ok': self._fail('virtserver response to %s: %s' % (cmd, r))
logging.debug('adt-virt << %s', r)
return l[1:]
def _vs_send(self, cmd):
if type(cmd) == type([]):
def maybe_quote(a):
if type(a) != type(()): return a
(a,) = a
return urllib.quote(a)
cmd = ' '.join(map(maybe_quote, cmd))
logging.debug('adt-virt >> %s', cmd)
print >>self._vs.stdin, cmd
return cmd.split(' ')[0]
def _command(self, cmd):
# argument forms: complete-command-string
# [arg, ...] where arg may be (arg,) to quote it
cmdp = self._vs_send(cmd)
self._vs.stdin.flush()
return self._awaitok(cmdp)
def _getfilecontents(self, filename):
try:
(_, tf) = create_temp_file()
self._command(['copyup', (filename,), (tf,)])
f = file(tf)
d = f.read()
f.close()
finally:
os.remove(tf)
return d
def create_temp_dir(self):
if self._vs is None:
logging.debug('adt-virt || %s' % self._cmdline)
self._vs = subprocess.Popen(self._cmdline, shell=True,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
self._awaitok('banner')
self._caps = self._command('capabilities')
def shutdown(self):
if self._vs is None: return
self._vs_send('quit')
self._vs.stdin.close()
self._vs.stdout.close()
self._vs.wait()
self._vs = None
def remove(self):
self._command('close')
dont_do_on_panic(self.panic_handler_id)
def _fail(self, m):
logging.error("adt-virt-* error: "+m)
panic()
def _open(self):
self._scratch = self._command('open')[0]
# this is a hack to make install_and_upgrade_between distros
# work; we pretend to save the chroot to a tarball but in
# fact we do nothing and then we can `restore' the `tarball' with
# adt-virt revert
def create_temp_tgz_file(self):
return self
def remove_temp_tgz_file(self, tgz):
if tgz is not self: self._fail('removing a tgz not supported')
# FIXME: anything else to do here?
def pack_into_tgz(self, tgz):
if tgz is not self: self._fail('packing into tgz not supported')
if not 'revert' in self._caps: self._fail('testbed cannot revert')
def unpack_from_tgz(self, tgz):
if tgz is not self: self._fail('unpacking from tgz not supported')
self._open()
def _execute(self, cmdl, tolerate_errors=False):
assert type(cmdl) == type([])
prefix = ['sh', '-ec', '''
LC_ALL=C
unset LANGUAGES
export LC_ALL
exec 2>&1
exec "$@"
''', '']
ca = ','.join(map(urllib.quote, prefix + cmdl))
stdout = '%s/cmd-stdout' % self._scratch
stderr = '%s/cmd-stderr-base' % self._scratch
cmd = ['execute', ca,
'/dev/null', (stdout,), (stderr,),
'/root', 'timeout=600']
es = int(self._command(cmd)[0])
if es and not tolerate_errors:
stderr_data = self._getfilecontents(stderr)
logging.error("Execution failed (status=%d): %s\n%s" %
(es, `cmdl`, indent_string(stderr_data)))
panic()
return (es, stdout, stderr)
def _execute_getoutput(self, cmdl):
(es, stdout, stderr) = self._execute(cmdl)
stderr_data = self._getfilecontents(stderr)
if es or stderr_data:
logging.error('Internal command failed (status=%d): %s\n%s' %
(es, `cmdl`, indent_string(stderr_data)))
panic()
(_, tf) = create_temp_file()
try:
self._command(['copyup', (stdout,), (tf,)])
except:
os.remove(tf)
raise
return tf
def run(self, command, ignore_errors=False):
cmdl = ['sh', '-ec', 'cd /\n' + ' '.join(command)]
(es, stdout, stderr) = self._execute(cmdl, tolerate_errors=True)
stdout_data = self._getfilecontents(stdout)
print >>sys.stderr, "VirtServ run", `command`, `cmdl`, '==>', `es`, `stdout`, `stderr`, '|', stdout_data
if es == 0 or ignore_errors: return (es, stdout_data)
stderr_data = self._getfilecontents(stderr)
logging.error('Command failed (status=%d): %s\n%s' %
(es, `command`, indent_string(stdout_data + stderr_data)))
panic()
def setup_minimal_chroot(self):
self._open()
def _tbpath(self, with_junk):
if not with_junk.startswith(self.name):
logging.error("Un-mangling testbed path `%s' but it does not"
"start with expected manglement `%s'" %
(with_junk, self.name))
panic()
return with_junk[len(self.name):]
def chmod(self, path, mode):
self._execute(['chmod', ('0%o' % mode), self._tbpath(path)])
def remove_files(self, paths):
self._execute(['rm', '--'] + map(self._tbpath, paths))
def copy_file(self, our_src, tb_dest):
self._command(['copydown', (our_src,),
(self._tbpath(tb_dest)+'/'+os.path.basename(our_src),)])
def create_file(self, path, data):
path = self._tbpath(path)
try:
(_, tf) = create_temp_file()
f = file(tf, 'w')
f.write(tf)
f.close()
self._command(['copydown', (tf,), (path,)])
finally:
os.remove(tf)
class DummyStat: pass
def save_meta_data(self):
mode_map = {
's': stat.S_IFSOCK,
'l': stat.S_IFLNK,
'f': stat.S_IFREG,
'b': stat.S_IFBLK,
'd': stat.S_IFDIR,
'c': stat.S_IFCHR,
'p': stat.S_IFIFO,
}
vdict = {}
tf = self._execute_getoutput(['find', '/', '-xdev', '-printf',
"%y %m %U %G %s %p %l \\n".replace(' ', '\\0')])
try:
f = file(tf)
while 1:
line = ''
while 1:
splut = line.split('\0')
if len(splut) == 8 and splut[7] == '\n': break
if len(splut) >= 8:
self._fail('aaargh wrong output from find: %s' %
urllib.quote(line), `splut`)
l = f.readline()
if not l:
if not line: break
self._fail('aargh missing final newline from find'
': %s, %s' % (`l`[0:200], `splut`[0:200]))
line += l
if not line: break
st = VirtServ.DummyStat()
st.st_mode = mode_map[splut[0]] | int(splut[1], 8)
(st.st_uid, st.st_gid, st.st_size) = map(int, splut[2:5])
vdict[splut[5]] = (st, splut[6])
f.close()
finally:
os.remove(tf)
return vdict
def get_files_owned_by_packages(self):
tf = self._execute_getoutput(['bash', '-ec', '''
cd /var/lib/dpkg/info
find . -name "*.list" -type f -print0 | \\
xargs -r0 egrep . /dev/null
test "${PIPESTATUS[*]}" = "0 0"
'''])
vdict = {}
try:
f = file(tf)
for l in f:
(lf, pathname) = l.rstrip('\n').split(':', 1)
assert lf.endswith('.list')
pkg = lf[:-5]
if pathname in vdict:
vdict[pathname].append(pkg)
else:
vdict[pathname] = [pkg]
f.close()
finally:
os.remove(tf)
return vdict
def check_for_broken_symlinks(self):
if not settings.check_broken_symlinks:
return
tf = self._execute_getoutput(['bash', '-ec', '''
find / -xdev -type l -print0 | \\
xargs -r0 -i'{}' \\
find '{}' -maxdepth 0 -follow -type l -ls
test "${PIPESTATUS[*]}" = "0 0"
'''])
try:
f = file(tf)
broken = False
for l in f:
logging.error("FAIL: Broken symlink: " + l)
broken = True
if broken: panic()
logging.debug("No broken symlinks found.")
finally:
os.remove(tf)
def check_for_no_processes(self): pass # ?!
def mount_proc(self): pass
def unmount_proc(self): pass
def selinux_enabled(enabled_test="/usr/sbin/selinuxenabled"):
if os.access(enabled_test, os.X_OK):
retval, output = run([enabled_test], ignore_errors=True)
if retval == 0:
return True
else:
return False
def objects_are_different(pair1, pair2):
"""Are filesystem objects different based on their meta data?"""
(m1, target1) = pair1
(m2, target2) = pair2
if (m1.st_mode != m2.st_mode or
m1.st_uid != m2.st_uid or
m1.st_gid != m2.st_gid or
target1 != target2):
return True
if stat.S_ISREG(m1.st_mode):
return m1.st_size != m2.st_size # or m1.st_mtime != m2.st_mtime
return False
def format_object_attributes(pair):
(st, target) = pair
ft = ""
if stat.S_ISDIR(st.st_mode):
ft += "d"
if stat.S_ISCHR(st.st_mode):
ft += "c"
if stat.S_ISBLK(st.st_mode):
ft += "b"
if stat.S_ISREG(st.st_mode):
ft += "-"
if stat.S_ISFIFO(st.st_mode):
ft += "p"
if stat.S_ISLNK(st.st_mode):
ft += "l"
if stat.S_ISSOCK(st.st_mode):
ft += "s"
res = "(%d, %d, %s %o, %d, %s)" % (
st.st_uid,
st.st_gid,
ft,
st.st_mode,
st.st_size,
target)
return res
def diff_meta_data(tree1, tree2):
"""Compare two dir trees and return list of new files (only in 'tree2'),
removed files (only in 'tree1'), and modified files."""
tree1 = tree1.copy()
tree2 = tree2.copy()
for name in settings.ignored_files:
if name in tree1:
del tree1[name]
if name in tree2:
del tree2[name]
for pattern in settings.ignored_patterns:
pat = re.compile(pattern)
for name in tree1.keys():
m = pat.search(name)
if m:
del tree1[name]
for name in tree2.keys():
m = pat.search(name)
if m:
del tree2[name]
modified = []
for name in tree1.keys()[:]:
if name in tree2:
if objects_are_different(tree1[name], tree2[name]):
logging.debug("Modified(uid, gid, mode, size, target): %s %s != %s" % \
(name, format_object_attributes(tree1[name]), format_object_attributes(tree2[name])))
modified.append((name, tree1[name]))
del tree1[name]
del tree2[name]
removed = [x for x in tree1.iteritems()]
new = [x for x in tree2.iteritems()]
# fix for #586793
# prune rc?.d symlinks renamed by insserv
pat1 = re.compile(r"^(/etc/rc.\.d/)[SK][0-9]{2}(.*)$")
for name1, data1 in removed[:]:
m = pat1.search(name1)
if m:
pat2 = re.compile(r"^" + m.group(1) + r"[SK][0-9]{2}" + m.group(2) + r"$")
for name2, data2 in new[:]:
m = pat2.search(name2)
if m:
logging.debug("File was renamed: %s\t=> %s" % (name1, name2))
removed.remove((name1, data1))
new.remove((name2, data2))
# this is again special casing due to the behaviour of a single package :(
# general tracking of moved files would be the better approach, probably.
return new, removed, modified
def file_list(meta_infos, file_owners):
"""Return list of indented filenames."""
meta_infos = meta_infos[:]
meta_infos.sort()
vlist = []
for name, data in meta_infos:
(st, target) = data
info = ""
if target is not None:
info = " -> %s" % target
vlist.append(" %s%s\t" % (name, info))
key = name
if key.endswith('/'):
key = key[:-1]
if key in file_owners:
vlist.append(" owned by: %s\n" % ", ".join(file_owners[key]))
else:
vlist.append(" not owned\n")
return "".join(vlist)
def offending_packages(meta_infos, file_owners):
"""Return a Set of offending packages."""
pkgset = set()
for name, data in meta_infos:
if name in file_owners:
for pkg in file_owners[name]:
pkgset.add(pkg)
return pkgset
def prune_files_list(files, depsfiles):
"""Remove elements from 'files' that are in 'depsfiles', and return the
list of removed elements.
"""
warn = []
depfiles_names = [x[0] for x in depsfiles]
for vfile in files[:]:
if vfile[0] in depfiles_names:
files.remove(vfile)
warn.append(vfile)
return warn
def diff_selections(chroot, selections):
"""Compare original and current package selection.
Return dict where dict[package_name] = original_status, that is,
the value in the dict is the state that the package needs to be
set to to restore original selections."""
changes = {}
current = chroot.get_selections()
for name, value in current.iteritems():
if name not in selections:
changes[name] = "purge"
elif selections[name] != current[name] and \
selections[name] in ["purge", "install"]:
changes[name] = selections[name]
for name, value in selections.iteritems():
if name not in current:
changes[name] = "install"
return changes
def get_package_names_from_package_files(package_files):
"""Return list of package names given list of package file names."""
vlist = []
for filename in package_files:
(status, output) = run(["dpkg", "--info", filename])
p = None
v = None
for line in [line.lstrip() for line in output.split("\n")]:
if line.startswith("Package:"):
p = line.split(":", 1)[1].strip()
if line.startswith("Version:"):
v = line.split(":", 1)[1].strip()
if p is not None:
if v is not None:
vlist.append(p + "=" + v)
else:
vlist.append(p)
return vlist
# Method to process a changes file, returning a list of all the .deb packages
# from the 'Files' stanza.
def process_changes(changes):
# Determine the path to the changes file, then check if it's readable.
dir_path = ""
changes_path = ""
if not os.path.dirname(changes):
changes_path = os.path.basename(changes)
else:
dir_path = os.path.dirname(changes) + "/"
changes_path = os.path.abspath(changes)
if not os.access(changes_path, os.R_OK):
logging.warn(changes_path + " is not readable. Skipping.")
return
# Determine the packages in the changes file through the 'Files' stanza.
field = 'Files'
pattern = re.compile(\
r'^'+field+r':' + r''' # The field we want the contents from
(.*?) # The contents of the field
\n([^ ]|$) # Start of a new field or EOF
''',
re.MULTILINE | re.DOTALL | re.VERBOSE)
f = open(changes_path)
file_text = f.read()
f.close()
matches = pattern.split(file_text)
# Append all the packages found in the changes file to a package list.
package_list = []
newline_p = re.compile('\n')
package_p = re.compile('.*?([^ ]+\.deb)$')
for line in newline_p.split(matches[1]):
if package_p.match(line):
package = dir_path + package_p.split(line)[1]
package_list.append(package)
# Return the list.
return package_list
def check_results(chroot, chroot_state, file_owners, deps_info=None):
"""Check that current chroot state matches 'chroot_state'.
If settings.warn_on_others is True and deps_info is not None, then only
print a warning rather than failing if the current chroot contains files
that are in deps_info but not in root_info. (In this case, deps_info
should be the result of chroot.save_meta_data() right after the
dependencies are installed, but before the actual packages to test are
installed.)
"""
root_info = chroot_state["tree"]
ok = True
if settings.check_broken_diversions:
(removed, added) = chroot.get_modified_diversions(chroot_state["diversions"])
if added:
logging.error("FAIL: Installed diversions (dpkg-divert) not removed by purge:\n%s" %
indent_string("\n".join(added)))
ok = False
if removed:
logging.error("FAIL: Existing diversions (dpkg-divert) removed/modified:\n%s" %
indent_string("\n".join(removed)))
ok = False
current_info = chroot.save_meta_data()
if settings.warn_on_others and deps_info is not None:
(new, removed, modified) = diff_meta_data(root_info, current_info)
(depsnew, depsremoved, depsmodified) = diff_meta_data(root_info,
deps_info)
warnnew = prune_files_list(new, depsnew)
warnremoved = prune_files_list(removed, depsremoved)
warnmodified = prune_files_list(modified, depsmodified)
else:
(new, removed, modified) = diff_meta_data(root_info, current_info)
if new:
if settings.warn_on_leftovers_after_purge:
logging.info("Warning: Package purging left files on system:\n" +
file_list(new, file_owners))
else:
logging.error("FAIL: Package purging left files on system:\n" +
file_list(new, file_owners))
ok = False
if removed:
logging.error("FAIL: After purging files have disappeared:\n" +
file_list(removed, file_owners))
ok = False
if modified:
logging.error("FAIL: After purging files have been modified:\n" +
file_list(modified, file_owners))
ok = False
if settings.warn_on_others and deps_info is not None:
if warnnew:
msg = ("Warning: Package purging left files on system:\n" +
file_list(warnnew, file_owners) + \
"These files seem to have been left by dependencies rather "
"than by packages\nbeing explicitly tested.\n")
logging.info(msg)
if warnremoved:
msg = ("After purging files have disappeared:\n" +
file_list(warnremoved, file_owners) +
"This seems to have been caused by dependencies rather "
"than by packages\nbeing explicitly tested.\n")
logging.info(msg)
if warnmodified:
msg = ("After purging files have been modified:\n" +
file_list(warnmodified, file_owners) +
"This seems to have been caused by dependencies rather "
"than by packages\nbeing explicitly tested.\n")
logging.info(msg)
return ok
def install_purge_test(chroot, chroot_state, package_files, packages, extra_packages):
"""Do an install-purge test. Return True if successful, False if not.
Assume 'root' is a directory already populated with a working
chroot, with packages in states given by 'selections'."""
deps_info = None
os.environ["PIUPARTS_TEST"] = "install"
chroot.run_scripts("pre_test")
# Install packages into the chroot.
os.environ["PIUPARTS_PHASE"] = "install"
chroot.enable_testdebs_repo()
chroot.check_for_no_processes(fail=True)
chroot.check_for_broken_symlinks()
chroot.run_scripts("pre_install")
chroot.install_packages([], extra_packages, with_scripts=False)
if settings.warn_on_others or settings.install_purge_install:
# Create a metapackage with dependencies from the given packages
if package_files:
control_infos = []
# We were given package files, so let's get the Depends and
# Conflicts directly from the .debs
for deb in package_files:
returncode, output = run(["dpkg", "-f", deb])
control = deb822.Deb822(output)
control_infos.append(control)
else:
# We have package names. Use apt to get all their control
# information.
apt_cache_args = ["apt-cache", "show"]
if os.environ["PIUPARTS_DISTRIBUTION"] in ["lenny"]:
# apt-cache in lenny does not accept version-qualified packages
apt_cache_args.extend([p.split("=", 1)[0].strip() for p in packages])
else:
apt_cache_args.extend(packages)
returncode, output = chroot.run(apt_cache_args)
control_infos = deb822.Deb822.iter_paragraphs(output.splitlines())
depends = []
conflicts = []
for control in control_infos:
if control.get("pre-depends"):
depends.append(control["pre-depends"])
if control.get("depends"):
depends.append(control["depends"])
if control.get("conflicts"):
conflicts.append(control["conflicts"])
all_depends = ", ".join(depends)
all_conflicts = ", ".join(conflicts)
metapackage = make_metapackage("piuparts-depends-dummy",
all_depends, all_conflicts)
cleanup_metapackage = lambda: shutil.rmtree(os.path.dirname(metapackage))
panic_handler_id = do_on_panic(cleanup_metapackage)
# Install the metapackage
chroot.install_package_files([metapackage], with_scripts=False)
# Now remove it
metapackagename = os.path.basename(metapackage)[:-4]
chroot.purge_packages([metapackagename])
cleanup_metapackage()
dont_do_on_panic(panic_handler_id)
# Save the file ownership information so we can tell which
# modifications were caused by the actual packages we are testing,
# rather than by their dependencies.
deps_info = chroot.save_meta_data()
if settings.install_purge_install:
# save chroot state with all deps installed
chroot_state_with_deps = {}
chroot_state_with_deps["tree"] = deps_info
chroot_state_with_deps["selections"] = chroot.get_selections()
chroot_state_with_deps["diversions"] = chroot.get_diversions()
chroot.check_for_no_processes()
chroot.check_for_broken_symlinks()
chroot.install_packages(package_files, packages, with_scripts=False)
chroot.run_scripts("post_install")
if settings.install_purge_install:
file_owners = chroot.get_files_owned_by_packages()
chroot.restore_selections(chroot_state_with_deps["selections"], packages)
logging.info("Validating chroot after purge")
chroot.check_debsums()
chroot.check_for_no_processes()
chroot.check_for_broken_symlinks()
if not check_results(chroot, chroot_state_with_deps, file_owners, deps_info=deps_info):
return False
logging.info("Reinstalling after purge")
chroot.install_packages(package_files, packages, with_scripts=True)
if settings.install_remove_install:
chroot.remove_packages(packages)
logging.info("Reinstalling after remove")
chroot.install_packages(package_files, packages, with_scripts=True)
chroot.check_for_no_processes()
chroot.check_for_broken_symlinks()
file_owners = chroot.get_files_owned_by_packages()
chroot.disable_testdebs_repo()
# Remove all packages from the chroot that weren't there initially.
chroot.restore_selections(chroot_state["selections"], packages)
chroot.check_for_no_processes(fail=True)
chroot.check_for_broken_symlinks()
return check_results(chroot, chroot_state, file_owners, deps_info=deps_info)
def install_upgrade_test(chroot, chroot_state, package_files, packages, old_packages):
"""Install old_packages via apt-get, then upgrade from package files.
Return True if successful, False if not."""
os.environ["PIUPARTS_TEST"] = "upgrade"
chroot.run_scripts("pre_test")
# First install via apt-get.
os.environ["PIUPARTS_PHASE"] = "install"
chroot.install_packages_by_name(old_packages)
chroot.check_for_no_processes()
chroot.check_for_broken_symlinks()
if settings.install_remove_install:
chroot.remove_packages(packages)
# Then from the package files.
os.environ["PIUPARTS_PHASE"] = "upgrade"
chroot.enable_testdebs_repo()
chroot.install_packages(package_files, packages)
chroot.check_for_no_processes()
chroot.check_for_broken_symlinks()
file_owners = chroot.get_files_owned_by_packages()
chroot.disable_testdebs_repo()
# Remove all packages from the chroot that weren't there initially.
chroot.restore_selections(chroot_state["selections"], packages)
chroot.check_for_no_processes(fail=True)
chroot.check_for_broken_symlinks()
return check_results(chroot, chroot_state, file_owners)
def save_meta_data(filename, chroot_state):
"""Save directory tree meta data into a file for fast access later."""
logging.debug("Saving chroot meta data to %s" % filename)
f = file(filename, "w")
pickle.dump(chroot_state, f)
f.close()
def load_meta_data(filename):
"""Load meta data saved by 'save_meta_data'."""
logging.debug("Loading chroot meta data from %s" % filename)
f = file(filename, "r")
chroot_state = pickle.load(f)
f.close()
return chroot_state
def install_and_upgrade_between_distros(package_files, packages_qualified):
"""Install package and upgrade it between distributions, then remove.
Return True if successful, False if not."""
# this function is a bit confusing at first, because of what it does by default:
# 1. create chroot with source distro
# 2. upgrade chroot to target distro
# 3. remove chroot and recreate chroot with source distro
# 4. install depends in chroot
# 5. install package in chroot
# 6. upgrade chroot to target distro
# 7. remove package and depends
# 8. compare results
#
# sounds silly, or?
# well, it is is a reasonable default (see below for why), but
#
# step 2+3 can be done differently by using --save-end-meta once and
# then --end-meta for all following runs - until the target distro
# changes again...
#
# Under normal circumstances the target distro can change anytime, ie. at
# the next mirror pulse, so unless the target distro is frozen, this is
# a reasonable default behaviour for distro upgrade tests, which are not
# done by default anyway.
os.environ["PIUPARTS_TEST"] = "distupgrade"
packages = [p.split("=", 1)[0].strip() for p in packages_qualified]
chroot = get_chroot()
chroot.create()
if settings.end_meta:
# load root_info and selections
chroot_state = load_meta_data(settings.end_meta)
else:
temp_tgz = None
if chroot.was_bootstrapped():
temp_tgz = chroot.create_temp_tgz_file()
panic_handler_id = do_on_panic(lambda: chroot.remove_temp_tgz_file(temp_tgz))
chroot.pack_into_tgz(temp_tgz)
chroot.upgrade_to_distros(settings.debian_distros[1:], [])
chroot.check_for_no_processes(fail=True)
# set root_info and selections
chroot_state = {}
chroot_state["tree"] = chroot.save_meta_data()
chroot_state["selections"] = chroot.get_selections()
chroot_state["diversions"] = chroot.get_diversions()
if settings.save_end_meta:
# save root_info and selections
save_meta_data(settings.save_end_meta, chroot_state)
chroot.remove()
# leave indication in logfile why we do what we do
logging.info("Notice: package selections and meta data from target distro saved, now starting over from source distro. See the description of --save-end-meta and --end-meta to learn why this is neccessary and how to possibly avoid it.")
chroot = get_chroot()
if temp_tgz is None:
chroot.create()
else:
chroot.create(temp_tgz)
chroot.remove_temp_tgz_file(temp_tgz)
dont_do_on_panic(panic_handler_id)
chroot.check_for_no_processes(fail=True)
chroot.run_scripts("pre_test")
os.environ["PIUPARTS_PHASE"] = "install"
distupgrade_packages = packages
known_packages = chroot.get_known_packages(packages + settings.extra_old_packages)
chroot.install_packages_by_name(known_packages)
if settings.install_remove_install:
chroot.remove_packages(packages)
distupgrade_packages = []
chroot.check_for_no_processes()
os.environ["PIUPARTS_PHASE"] = "distupgrade"
chroot.upgrade_to_distros(settings.debian_distros[1:-1], distupgrade_packages)
if settings.distupgrade_to_testdebs:
chroot.enable_testdebs_repo(update=False)
chroot.upgrade_to_distros(settings.debian_distros[-1:], distupgrade_packages)
chroot.check_for_no_processes()
os.environ["PIUPARTS_PHASE"] = "upgrade"
if not settings.distupgrade_to_testdebs:
chroot.enable_testdebs_repo()
chroot.install_packages(package_files, [p for p in packages_qualified if not p.endswith("=None")])
chroot.disable_testdebs_repo()
chroot.check_for_no_processes()
file_owners = chroot.get_files_owned_by_packages()
# Remove all packages from the chroot that weren't in the reference chroot.
chroot.restore_selections(chroot_state["selections"], packages_qualified)
chroot.check_for_no_processes(fail=True)
result = check_results(chroot, chroot_state, file_owners)
chroot.remove()
return result
def parse_mirror_spec(str, defaultcomponents=[]):
"""Parse a mirror specification from the --mirror option argument.
Return (mirror, componentslist)."""
parts = str.split()
return parts[0], parts[1:] or defaultcomponents[:]
def find_default_debian_mirrors():
"""Find the default Debian mirrors."""
mirrors = []
try:
f = file("/etc/apt/sources.list", "r")
for line in f:
parts = line.split()
if len(parts) > 2 and parts[0] == "deb":
mirrors.append((parts[1], parts[3:]))
break # Only use the first one, at least for now.
f.close()
except IOError:
return None
return mirrors
def forget_ignores(option, opt, value, parser, *args, **kwargs):
settings.bindmounts = []
parser.values.ignore = []
parser.values.ignore_regex = []
settings.ignored_files = []
settings.ignored_patterns = []
def set_basetgz_to_pbuilder(option, opt, value, parser, *args, **kwargs):
parser.values.basetgz = "/var/cache/pbuilder/base.tgz"
def parse_command_line():
"""Parse the command line, change global settings, return non-options."""
parser = optparse.OptionParser(usage="%prog [options] package ...",
version="piuparts %s" % VERSION)
parser.add_option("-a", "--apt", action="store_true", default=False,
help="Command line arguments are package names " +
"to be installed via apt.")
parser.add_option("--arch", metavar="ARCH", action="store",
help="Create chroot and run tests for (non-default) architecture ARCH.")
parser.add_option("--adt-virt",
metavar='CMDLINE', default=None,
help="Use CMDLINE via autopkgtest (adt-virt-*)"
" protocol instead of managing a chroot.")
parser.add_option("-b", "--basetgz", metavar="TARBALL",
help="Use TARBALL as the contents of the initial " +
"chroot, instead of building a new one with " +
"debootstrap.")
parser.add_option("--bindmount", action="append", metavar="DIR",
default=[],
help="Directory to be bind-mounted inside the chroot.")
parser.add_option("-d", "--distribution", action="append", metavar="NAME",
help="Which Debian distribution to use: a code name " +
"(for example squeeze, wheezy, sid) or experimental. The " +
"default is sid (=unstable).")
parser.add_option("-D", "--defaults", action="store",
help="Choose which set of defaults to use "
"(debian/ubuntu).")
parser.add_option("--debfoster-options",
default="-o MaxPriority=required -o UseRecommends=no -f -n apt debfoster",
help="Run debfoster with different parameters (default: -o MaxPriority=required -o UseRecommends=no -f -n apt debfoster).")
parser.add_option("--no-eatmydata",
default=False,
action='store_true',
help="Default is to use libeatmydata in the chroot")
parser.add_option("--dpkg-noforce-unsafe-io",
default=False,
action='store_true',
help="Default is to run dpkg with --force-unsafe-io option, which causes dpkg to skip certain file system syncs known to cause substantial performance degradation on some filesystems. This option turns that off and dpkg will use safe I/O operations.")
parser.add_option("--dpkg-force-confdef",
default=False,
action='store_true',
help="Make dpkg use --force-confdef, which lets dpkg always choose the default action when a modified conffile is found. This option will make piuparts ignore errors it was designed to report and therefore should only be used to hide problems in depending packages. (See #466118.)")
parser.add_option("--do-not-verify-signatures", default=False,
action='store_true',
help="Do not verify signatures from the Release files when running debootstrap.")
parser.add_option("--allow-database", default=False,
action='store_true',
help="Allow database servers (MySQL, PostgreSQL) to be started in the chroot.")
parser.add_option("--distupgrade-to-testdebs", default=False,
action='store_true',
help="Use the testdebs repository as distupgrade target.")
parser.add_option("-e", "--existing-chroot", metavar="DIR",
help="Use DIR as the contents of the initial " +
"chroot, instead of building a new one with " +
"debootstrap")
parser.add_option("-i", "--ignore", action="append", metavar="FILENAME",
default=[],
help="Add FILENAME to list of filenames to be " +
"ignored when comparing changes to chroot.")
parser.add_option("-I", "--ignore-regex", action="append",
metavar="REGEX", default=[],
help="Add REGEX to list of Perl compatible regular " +
"expressions for filenames to be " +
"ignored when comparing changes to chroot.")
parser.add_option("--install-recommends",
action="store_true", default=False,
help="Enable the installation of Recommends.")
parser.add_option("-k", "--keep-tmpdir",
action="store_true", default=False,
help="Don't remove the temporary directory for the " +
"chroot when the program ends.")
parser.add_option("-K", "--keyring", action="store", metavar="FILE",
help="Use FILE as the keyring to use with debootstrap when creating chroots.")
parser.add_option("--keep-sources-list",
action="store_true", default=False,
help="Don't modify the chroot's " +
"etc/apt/sources.list (only makes sense " +
"with --basetgz).")
parser.add_option("-l", "--log-file", metavar="FILENAME",
help="Write log file to FILENAME in addition to " +
"the standard output.")
parser.add_option("--list-installed-files",
action="store_true", default=False,
help="List files added to the chroot after the " +
"installation of the package.")
parser.add_option("--lvm-volume", metavar="LVM-VOL", action="store",
help="Use LVM-VOL as source for the chroot, instead of building " +
"a new one with debootstrap. This creates a snapshot of the " +
"given LVM volume and mounts it to the chroot path")
parser.add_option("--lvm-snapshot-size", metavar="SNAPSHOT-SIZE", action="store",
default="1G", help="Use SNAPSHOT-SIZE as snapshot size when creating " +
"a new LVM snapshot (default: 1G)")
parser.add_option("--schroot", metavar="SCHROOT-NAME", action="store",
help="Use schroot session named SCHROOT-NAME for the chroot, instead of building " +
"a new one with debootstrap.")
parser.add_option("-m", "--mirror", action="append", metavar="URL",
default=[],
help="Which Debian mirror to use.")
parser.add_option("--extra-repo", action="append",
default=[],
help="Additional (unparsed) lines to be appended to sources.list, e.g. " +
"'deb ' or 'deb file:// ./'")
parser.add_option("--testdebs-repo",
help="A repository that contains the packages to be tested, e.g. " +
"'deb ...' or 'deb file:// ./'," +
"plain URLs or local paths are permitted, too.")
parser.add_option("--no-adequate",
default=False,
action='store_true',
help="Don't run adequate after installation.")
parser.add_option("--no-diversions", action="store_true",
default=False,
help="Don't check for broken diversions.")
parser.add_option("-n", "--no-ignores", action="callback",
callback=forget_ignores,
help="Forget all ignores set so far, including " +
"built-in ones.")
parser.add_option("-N", "--no-symlinks", action="store_true",
default=False,
help="Don't check for broken symlinks.")
parser.add_option("--no-upgrade-test",
action="store_true", default=False,
help="Skip testing the upgrade from an existing version " +
"in the archive.")
parser.add_option("--no-install-purge-test",
action="store_true", default=False,
help="Skip install and purge test.")
parser.add_option("--install-purge-install",
action="store_true", default=False,
help="Purge package after installation and reinstall.")
parser.add_option("--install-remove-install",
action="store_true", default=False,
help="Remove package after installation and reinstall. For testing installation in config-files-remaining state.")
parser.add_option("--extra-old-packages",
action="append", default=[],
help="Install these additional packages along with the old packages from the archive. " +
"Useful to test Conflicts/Replaces of packages that will disappear during the update. " +
"Takes a comma separated list of package names and can be given multiple times. " +
"For install/purge tests these packages will be installed before the package that is to be tested.")
parser.add_option("-p", "--pbuilder", action="callback",
callback=set_basetgz_to_pbuilder,
help="Use /var/cache/pbuilder/base.tgz as the base " +
"tarball.")
parser.add_option("--pedantic-purge-test",
action="store_true", default=False,
help="Be pedantic when checking if a purged package leaves files behind. If this option is not set, files left in /tmp are ignored.")
parser.add_option("--proxy", metavar="URL",
help="Use the proxy at URL for accessing the mirrors.")
parser.add_option("-s", "--save", metavar="FILENAME",
help="Save the chroot into FILENAME.")
parser.add_option("-B", "--end-meta", metavar="FILE",
help="Load chroot package selection and file meta data from FILE. See the function install_and_upgrade_between_distros() in piuparts.py for defaults. Mostly useful for large scale distro upgrade tests.")
parser.add_option("-S", "--save-end-meta", metavar="FILE",
help="Save chroot package selection and file meta data in FILE for later use. See the function install_and_upgrade_between_distros() in piuparts.py for defaults. Mostly useful for large scale distro upgrade tests.")
parser.add_option("--single-changes-list", default=False,
action="store_true",
help="test all packages from all changes files together.")
parser.add_option("--skip-cronfiles-test",
action="store_true", default=False,
help="Skip testing the output from the cron files.")
parser.add_option("--skip-logrotatefiles-test",
action="store_true", default=False,
help="Skip testing the output from the logrotate files.")
parser.add_option("--skip-minimize",
action="store_true", default=True,
help="Skip minimize chroot step. This is the default now.")
parser.add_option("--minimize",
action="store_true", default=False,
help="Minimize chroot with debfoster. This used to be the default until #539142 was fixed.")
parser.add_option("--scriptsdir", metavar="DIR",
action="append", default=[],
help="Directory where are placed the custom scripts. Can be given multiple times.")
parser.add_option("-t", "--tmpdir", metavar="DIR",
help="Use DIR for temporary storage. Default is " +
"$TMPDIR or /tmp.")
parser.add_option("-v", "--verbose",
action="store_true", default=False,
help="No meaning anymore.")
parser.add_option("--warn-on-others",
action="store_true", default=False,
help="Print a warning rather than failing if "
"files are left behind, modified, or removed "
"by a package that was not given on the "
"command-line. Behavior with multiple packages "
"given could be problematic, particularly if the "
"dependency tree of one package in the list "
"includes another in the list. Therefore, it is "
"recommended to use this option with one package "
"at a time.")
parser.add_option("--warn-on-leftovers-after-purge",
action="store_true", default=False,
help="Print a warning rather than failing if "
"files are left behind after purge.")
parser.add_option("--warn-on-debsums-errors",
action="store_true", default=False,
help="Print a warning rather than failing if "
"debsums reports modified files.")
parser.add_option("--fail-if-inadequate",
action="store_true", default=False,
help="Fail on inadequate results from running adequate.")
parser.add_option("--fail-on-broken-symlinks", action="store_true",
default=False,
help="Fail if broken symlinks are detected.")
parser.add_option("--log-level", action="store", metavar='LEVEL',
default="dump",
help="Displays messages from LEVEL level, possible values are: error, info, dump, debug. The default is dump.")
(opts, args) = parser.parse_args()
settings.defaults = opts.defaults
defaults = DefaultsFactory().new_defaults()
settings.tmpdir = opts.tmpdir
settings.keep_tmpdir = opts.keep_tmpdir
settings.single_changes_list = opts.single_changes_list
settings.args_are_package_files = not opts.apt
# distro setup
settings.proxy = opts.proxy
if settings.proxy:
os.environ["http_proxy"] = settings.proxy
settings.debian_mirrors = [parse_mirror_spec(x, defaults.get_components())
for x in opts.mirror]
settings.extra_repos = opts.extra_repo
settings.testdebs_repo = opts.testdebs_repo
settings.debian_distros = opts.distribution
settings.keep_sources_list = opts.keep_sources_list
if opts.keyring:
settings.keyring = opts.keyring
else:
settings.keyring = defaults.get_keyring()
settings.do_not_verify_signatures = opts.do_not_verify_signatures
if settings.do_not_verify_signatures:
settings.keyringoption=""
settings.apt_unauthenticated="Yes"
else:
settings.keyringoption="--keyring=%s" % settings.keyring
settings.apt_unauthenticated="No"
settings.install_recommends = opts.install_recommends
settings.eatmydata = not opts.no_eatmydata
settings.dpkg_force_unsafe_io = not opts.dpkg_noforce_unsafe_io
settings.dpkg_force_confdef = opts.dpkg_force_confdef
settings.scriptsdirs = opts.scriptsdir
settings.bindmounts += opts.bindmount
settings.allow_database = opts.allow_database
# chroot setup
settings.arch = opts.arch
settings.basetgz = opts.basetgz
settings.savetgz = opts.save
settings.lvm_volume = opts.lvm_volume
settings.lvm_snapshot_size = opts.lvm_snapshot_size
settings.existing_chroot = opts.existing_chroot
settings.schroot = opts.schroot
settings.end_meta = opts.end_meta
settings.save_end_meta = opts.save_end_meta
settings.skip_minimize = opts.skip_minimize
settings.minimize = opts.minimize
if settings.minimize:
settings.skip_minimize = False
settings.debfoster_options = opts.debfoster_options.split()
# tests and checks
settings.no_install_purge_test = opts.no_install_purge_test
settings.no_upgrade_test = opts.no_upgrade_test
settings.distupgrade_to_testdebs = opts.distupgrade_to_testdebs
settings.install_purge_install = opts.install_purge_install
settings.install_remove_install = opts.install_remove_install
settings.list_installed_files = opts.list_installed_files
[settings.extra_old_packages.extend([i.strip() for i in csv.split(",")]) for csv in opts.extra_old_packages]
settings.skip_cronfiles_test = opts.skip_cronfiles_test
settings.skip_logrotatefiles_test = opts.skip_logrotatefiles_test
settings.adequate = not opts.no_adequate
settings.check_broken_diversions = not opts.no_diversions
settings.check_broken_symlinks = not opts.no_symlinks
settings.warn_broken_symlinks = not opts.fail_on_broken_symlinks
settings.warn_on_others = opts.warn_on_others
settings.warn_on_leftovers_after_purge = opts.warn_on_leftovers_after_purge
settings.warn_on_debsums_errors = opts.warn_on_debsums_errors
settings.warn_if_inadequate = not opts.fail_if_inadequate
settings.pedantic_purge_test = opts.pedantic_purge_test
settings.ignored_files += opts.ignore
settings.ignored_patterns += opts.ignore_regex
if not settings.pedantic_purge_test:
settings.ignored_patterns += settings.non_pedantic_ignore_patterns
if opts.adt_virt is None:
settings.adt_virt = None
else:
settings.adt_virt = VirtServ(opts.adt_virt)
log_file_name = opts.log_file
if opts.log_level == "error":
setup_logging(logging.ERROR, log_file_name)
elif opts.log_level == "info":
setup_logging(logging.INFO, log_file_name)
elif opts.log_level == "debug":
setup_logging(logging.DEBUG, log_file_name)
else:
setup_logging(DUMP, log_file_name)
exitcode = None
if not settings.tmpdir:
if "TMPDIR" in os.environ:
settings.tmpdir = os.environ["TMPDIR"]
else:
settings.tmpdir = "/tmp"
if not os.path.isdir(settings.tmpdir):
logging.error("Temporary directory is not a directory: %s" %
settings.tmpdir)
panic()
for sdir in settings.scriptsdirs:
if not os.path.isdir(sdir):
logging.error("Scripts directory is not a directory: %s" % sdir)
panic()
if not settings.debian_distros:
settings.debian_distros = defaults.get_distribution()
if not settings.debian_mirrors:
if opts.defaults:
settings.debian_mirrors = defaults.get_mirror()
else:
settings.debian_mirrors = find_default_debian_mirrors()
if not settings.debian_mirrors:
settings.debian_mirrors = defaults.get_mirror()
settings.distro_config = piupartslib.conf.DistroConfig(
DISTRO_CONFIG_FILE, settings.debian_mirrors[0][0])
if settings.keep_sources_list and \
(not settings.basetgz or len(settings.debian_distros) > 1):
logging.error("--keep-sources-list only makes sense with --basetgz "
"and only one distribution")
exitcode = 1
if not args:
logging.error("Need command line arguments: " +
"names of packages or package files")
exitcode = 1
settings.testobjects = args
if exitcode is not None:
sys.exit(exitcode)
return args
def get_chroot():
if settings.adt_virt is None: return Chroot()
return settings.adt_virt
# Process the packages given in a list
def process_packages(package_list):
# Find the names of packages.
if settings.args_are_package_files:
packages = get_package_names_from_package_files(package_list)
package_files = package_list
else:
packages = package_list
package_files = []
if len(settings.debian_distros) == 1:
chroot = get_chroot()
chroot.create()
chroot_state = {}
chroot_state["tree"] = chroot.save_meta_data()
chroot_state["selections"] = chroot.get_selections()
chroot_state["diversions"] = chroot.get_diversions()
if not settings.no_install_purge_test:
extra_packages = chroot.get_known_packages(settings.extra_old_packages)
if not install_purge_test(chroot, chroot_state,
package_files, packages, extra_packages):
logging.error("FAIL: Installation and purging test.")
panic()
logging.info("PASS: Installation and purging test.")
if not settings.no_upgrade_test:
if not settings.args_are_package_files and not settings.testdebs_repo:
logging.info("Can't test upgrades: -a or --apt option used.")
else:
packages_to_query = [p.split("=", 1)[0].strip() for p in packages]
packages_to_query.extend(settings.extra_old_packages)
known_packages = chroot.get_known_packages(packages_to_query)
if not known_packages:
logging.info("Can't test upgrade: packages not known by apt-get.")
elif install_upgrade_test(chroot, chroot_state, package_files,
packages, known_packages):
logging.info("PASS: Installation, upgrade and purging tests.")
else:
logging.error("FAIL: Installation, upgrade and purging tests.")
panic()
chroot.remove()
else:
if install_and_upgrade_between_distros(package_files, packages):
logging.info("PASS: Upgrading between Debian distributions.")
else:
logging.error("FAIL: Upgrading between Debian distributions.")
panic()
if settings.adt_virt is not None: settings.adt_virt.shutdown()
def main():
"""Main program. But you knew that."""
args = parse_command_line()
# check if user has root privileges
if os.getuid():
print 'You need to be root to use piuparts.'
sys.exit(1)
logging.info("-" * 78)
logging.info("To quickly glance what went wrong, scroll down to the bottom of this logfile.")
logging.info("FAQ available at http://wiki.debian.org/piuparts/FAQ")
logging.info("The FAQ also explains how to contact us in case you think piuparts is wrong.")
logging.info("-" * 78)
logging.info("piuparts version %s starting up." % VERSION)
logging.info("Command line arguments: %s" % " ".join(quote_spaces(sys.argv)))
logging.info("Running on: %s %s %s %s %s" % os.uname())
# Make sure debconf does not ask questions and stop everything.
# Packages that don't use debconf will lose.
os.environ["DEBIAN_FRONTEND"] = "noninteractive"
if "DISPLAY" in os.environ:
del os.environ["DISPLAY"]
changes_packages_list = []
regular_packages_list = []
changes_p = re.compile('.*\.changes$')
for arg in args:
if changes_p.match(arg):
package_list = process_changes(arg)
if settings.single_changes_list:
for package in package_list:
regular_packages_list.append(package)
else:
changes_packages_list.append(package_list)
else:
regular_packages_list.append(arg)
if changes_packages_list:
for package_list in changes_packages_list:
process_packages(package_list)
if regular_packages_list:
process_packages(regular_packages_list)
logging.info("PASS: All tests.")
logging.info("piuparts run ends.")
if __name__ == "__main__":
try:
if sys.argv[1:] == ["unittest"]:
del sys.argv[1]
unittest.main()
else:
main()
except KeyboardInterrupt:
print ''
print 'Piuparts interrupted by the user, exiting...'
panic(1)
sys.exit(1)
# vi:set et ts=4 sw=4 :
piuparts-0.56/piuparts_slave_join.8.txt 0000664 0000000 0000000 00000002017 12155061511 015167 0 ustar piuparts_slave_join(8)
======================
:doctype: manpage
NAME
----
piuparts_slave_join - join a running piuparts-slave screen session
SYNOPSIS
--------
*piuparts_slave_join*
DESCRIPTION
-----------
*piuparts_slave_join* joins a running piuparts-slave session in screen.
OPTIONS
-------
There are no options to this command.
ENVIRONMENT
-----------
Running piuparts in master-slave mode requires configuration in _/etc/piuparts_.
NOTES
-----
Make sure to also read */usr/share/doc/piuparts-master/README_server.txt*.
See *screen*(1) for full instructions on how to navigate between displays. In short:
*Ctrl-A *::
Navigates to display . Slaves are found at screens 1 through 'slave-count', defined in the piuparts configuration file.
*Ctrl-A d*::
Detatch from *screen*. The session can be re-established with *piuparts-slave-join*.
SEE ALSO
--------
*piuparts*(1), *piuparts_slave_run*(8), *screen*(1)
AUTHOR
------
Holger Levsen (holger@layer-acht.org)
DATE
----
2013-05-27
// vim: set filetype=asciidoc:
piuparts-0.56/unittests.py 0000664 0000000 0000000 00000005523 12144653075 012633 0 ustar # -*- coding: utf-8 -*-
import os
import StringIO
import unittest
import piupartslib.packagesdb
import piupartslib.dependencyparser
class DependencyParserTests(unittest.TestCase):
"""Tests for module dependencyparser."""
def parse(self, str):
parser = piupartslib.dependencyparser.DependencyParser(str)
deps = parser.get_dependencies()
names = []
for dep in deps:
names.append([])
for simpledep in dep:
names[-1].append(simpledep.name)
return deps, names
def testEmpty(self):
deps, names = self.parse("")
self.failUnlessEqual(deps, [])
def testSingle(self):
deps, names = self.parse("foo")
self.failUnlessEqual(names, [["foo"]])
def testTwo(self):
deps, names = self.parse("foo, bar")
self.failUnlessEqual(names, [["foo"], ["bar"]])
def testAlternatives(self):
deps, names = self.parse("foo, bar | foobar")
self.failUnlessEqual(names, [["foo"], ["bar", "foobar"]])
class FakeLogDB(piupartslib.packagesdb.LogDB):
"""A fake version of the LogDB class, for testing
This version simulates filesystem actions so that there is no need
to do actual I/O. Cleaner, although not quite as thorough.
"""
def __init__(self):
self.dict = {
"pass": [],
"fail": [],
"untestable": [],
"reserved": [],
"bugged": [],
"affected": [],
}
def listdir(self, dirname):
return self.dict[dirname]
def _parse(self, pathname):
return os.path.dirname(pathname), os.path.basename(pathname)
def exists(self, pathname):
vdir, base = self._parse(pathname)
return base in self.dict[vdir]
def open_file(self, pathname, mode):
vdir, base = self._parse(pathname)
self.dict[vdir].append(base)
return StringIO.StringIO()
def remove_file(self, pathname):
vdir, base = self._parse(pathname)
if base in self.dict[vdir]:
del self.dict[vdir]
def create(self, subdir, package, version, contents):
return True
class PackagesDbTests(unittest.TestCase):
def new_db(self, packages_file_contents):
db = piupartslib.packagesdb.PackagesDB(FakeLogDB())
db.read_packages_file(StringIO.StringIO(packages_file_contents))
return db
def reserve(self, packages_file_contents):
db = self.new_db(packages_file_contents)
return db.reserve_package()
def testNoPackages(self):
p = self.reserve("")
self.failUnlessEqual(p, None)
def testNoDeps(self):
p = self.reserve("""\
Package: foo
Version: 1.0-1
""")
self.failIfEqual(p, None)
self.failUnlessEqual(p["Package"], "foo")
if __name__ == "__main__":
unittest.main()
# vi:set et ts=4 sw=4 :
piuparts-0.56/known_problems/ 0000775 0000000 0000000 00000000000 12253565512 013250 5 ustar piuparts-0.56/known_problems/unknown_inadequate_issue.conf 0000664 0000000 0000000 00000001176 12172224537 021232 0 ustar #
# detect packages which have the string "Found unknown tags running adequate" in their logs
#
PATTERN='(FAIL|WARN): Found unknown tags running adequate'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER='Packages which have logs with the string "Found unknown tags running adequate"'
HELPTEXT='
Sometimes new types of problems are detected by adequate , which classifies them using tags. When this happens these new tags need to be made known to piuparts. Please notify piuparts-devel@lists.alioth.debian.org .
'
piuparts-0.56/known_problems/owned_files_after_purge_issue.conf 0000664 0000000 0000000 00000001062 12145710343 022201 0 ustar #
# detect packages with owned files after purge (policy 6.8)
#
PATTERN='owned by:'
WHERE='pass'
ISSUE=1
HEADER='Packages with owned files after purge'
HELPTEXT='
WARNING: this list might contain false positives.
#316521: dpkg: incomplete cleanup of empty directories
Packages need to remove owned files after purge, see
http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.3
'
piuparts-0.56/known_problems/initdscript_lsb_header_issue.conf 0000664 0000000 0000000 00000002561 12145710343 022031 0 ustar #
# detect packages with an update-rc.d warning
#
PATTERN='update-rc.d: warning.*do not match LSB'
WHERE='fail bugged affected pass'
ISSUE=1
HEADER='Packages with logs with the string "update-rc.d: warning.*do not match LSB"'
HELPTEXT='
Some packages have inconsistency between the init.d script headers
used with dependency based boot sequencing and the runlevels specified
on the update-rc.d command line and used by the legacy boot ordering.
Such inconsistency is most likely a bug in the package, as the two
ways of ordering init.d scripts should enable and disable the scripts
in the same runlevels while Debian migrate to dependency based boot
sequencing.
Such inconsinstency is reported like this when a postinst script call
update-rc.d
update-rc.d: warning: initdscript start runlevel arguments (2 3 4 5) do not match LSB Default-Start values (S)
update-rc.d: warning: initdscript stop runlevel arguments (0 1 6) do not match LSB Default-Stop values (none)
Such reports are most likely bugs in the package calling update-rc.d,
and should be reported and fixed in the individual packages.
See the paragraph "How to solve migration problems" at http://wiki.debian.org/LSBInitScripts/DependencyBasedBoot for information how to fix these issues.
'
piuparts-0.56/known_problems/post_installation_script_error.conf 0000664 0000000 0000000 00000000536 12145710343 022460 0 ustar #
# detect packages which failed because post-installation maintainer script failed
#
PATTERN='subprocess .*post-installation script returned error'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because post-installation maintainer script failed'
HELPTEXT='
This is a somewhat unclassified failure at the moment.
'
piuparts-0.56/known_problems/obsolete_conffiles_error.conf 0000664 0000000 0000000 00000001007 12145710343 021164 0 ustar #
# detect packages that leave obsolete conffiles after upgrades
#
PATTERN='OBSOLETE CONFFILE'
WHERE='fail bugged affected'
ISSUE=1
HEADER='Packages leaving obsolete conffiles after upgrade'
HELPTEXT='
Packages that leave obsolete conffiles after upgrade.
Using
dpkg-maintscript-helper rm_conffile via dh_installdeb package.maintscript files is the recommended way to clean them up.
There may be false positives, e.g. if a conffile was converted to a maintainer script managed configuration file.
'
piuparts-0.56/known_problems/ldd_inadequate_issue.conf 0000664 0000000 0000000 00000000567 12175701021 020270 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* ldd'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'ldd' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'ldd' which indicates a bug.
"
piuparts-0.56/known_problems/missing_md5sums_issue.conf 0000664 0000000 0000000 00000000367 12155061511 020452 0 ustar #
# detect packages that ship files without md5sums
#
PATTERN='(FILE WITHOUT MD5SUM|MD5SUM FILE NOT FOUND)'
WHERE='pass'
ISSUE=1
HEADER='Packages containing files without md5sum'
HELPTEXT='
Packages that ship files that have no md5sum.
'
piuparts-0.56/known_problems/missing-symbol-version-information_inadequate_issue.conf 0000664 0000000 0000000 00000000724 12253565512 026514 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* missing-symbol-version-information'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'missing-symbol-version-information' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'missing-symbol-version-information' which indicates a bug.
"
piuparts-0.56/known_problems/broken_symlinks_issue.conf 0000664 0000000 0000000 00000001015 12145710343 020527 0 ustar #
# detect packages which have the string "Broken symlinks" in their logs
#
PATTERN='(WARN|FAIL): Broken symlink'
WHERE='pass'
ISSUE=1
HEADER='Packages which have logs with the string "Broken symlinks"'
HELPTEXT='
This is clearly an error, but as there are too many of this kind, piuparts can be configured to not fail if it detects broken symlinks. Another option is not to test for broken symlinks. See the piuparts manpage for details.
'
piuparts-0.56/known_problems/obsolete_conffiles_issue.conf 0000664 0000000 0000000 00000000767 12145710343 021177 0 ustar #
# detect packages that leave obsolete conffiles after upgrades
#
PATTERN='OBSOLETE CONFFILE'
WHERE='pass'
ISSUE=1
HEADER='Packages leaving obsolete conffiles after upgrade'
HELPTEXT='
Packages that leave obsolete conffiles after upgrade.
Using
dpkg-maintscript-helper rm_conffile via dh_installdeb package.maintscript files is the recommended way to clean them up.
There may be false positives, e.g. if a conffile was converted to a maintainer script managed configuration file.
'
piuparts-0.56/known_problems/installs_over_symlink_error.conf 0000664 0000000 0000000 00000001152 12145710343 021753 0 ustar #
# detect packages which have the string "dirname part contains a symlink" in their logs
#
PATTERN='silently overwrites files via directory symlinks|dirname part contains a symlink'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages that install something over existing symlinks'
HELPTEXT='
Installing anything over a symlink opens a can of worms -
this causes problems on upgrades while switching between directory and symlink
or if the symlink is ever changed.
Piuparts looks at all $pathname known to dpkg and checks for
$(dirname $pathname) != $(readlink $(dirname $pathname))
'
piuparts-0.56/known_problems/py_file_not_bytecompiled_inadequate_issue.conf 0000664 0000000 0000000 00000000666 12175701021 024574 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* py-file-not-bytecompiled'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'py-file-not-bytecompiled' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'py-file-not-bytecompiled' which indicates a bug.
"
piuparts-0.56/known_problems/library_not_found_inadequate_issue.conf 0000664 0000000 0000000 00000000641 12175701021 023235 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* library-not-found'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'library-not-found' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'library-not-found' which indicates a bug.
"
piuparts-0.56/known_problems/db_setup_issue.conf 0000664 0000000 0000000 00000001424 12145710343 017127 0 ustar #
# detect packages with failed to install due to problems configuring the db - see #595652
#
PATTERN='(warning: database package not installed|dbconfig-common: .* configure: (aborted|noninteractive fail).|updating database schema for .*...command failed with code 0|psql: could not connect to server: No such file or directory|DBI connect.* failed: could not connect to server|pg_pconnect\(\): Unable to connect to PostgreSQL server|Unable to connect to MySQL server|unable to connect to mysql server)'
WHERE='pass'
ISSUE=1
HEADER='Packages with failed logs because installation failed because no database could be connected.'
HELPTEXT='
Just how buggy these packages really are is discussed in #595652 at the moment.
'
piuparts-0.56/known_problems/pre_removal_script_error.conf 0000664 0000000 0000000 00000000514 12145710343 021221 0 ustar #
# detect packages which failed because pre-removal maintainer script failed
#
PATTERN='subprocess .*pre-removal script returned error'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because pre-removal maintainer script failed'
HELPTEXT='
This is a somewhat unclassified failure at the moment.
'
piuparts-0.56/known_problems/maintainer_script_issue.conf 0000664 0000000 0000000 00000000547 12145710343 021042 0 ustar #
# detect packages in successfully-tested state that had maintainer script failures
#
PATTERN='subprocess .*(pre|post)-(installation|removal) script returned error'
WHERE='pass'
ISSUE=0
HEADER='Packages in state successfully-tested but logfile contains a maintainer script failure'
HELPTEXT='
This is a somewhat unclassified issue at the moment.
'
piuparts-0.56/known_problems/files_in_usr_local_error.conf 0000664 0000000 0000000 00000000664 12145710343 021163 0 ustar #
# detect packages which leave stuff in /usr/local (see policy 9.1.2)
#
PATTERN='usr/local.+not owned'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with files and/or directories in /usr/local/ after purge'
HELPTEXT='
This is a violation of policy 9.1.2: see
http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.1.2 .
'
piuparts-0.56/known_problems/bin_or_sbin_binary_requires_usr_lib_library_inadequate_issue.conf 0000664 0000000 0000000 00000000757 12175701021 030537 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* bin-or-sbin-binary-requires-usr-lib-library'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'bin-or-sbin-binary-requires-usr-lib-library' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'bin-or-sbin-binary-requires-usr-lib-library' which indicates a bug.
"
piuparts-0.56/known_problems/pre_installation_script_error.conf 0000664 0000000 0000000 00000000533 12145710343 022256 0 ustar #
# detect packages which failed because pre-installation maintainer script failed
#
PATTERN='subprocess .*pre-installation script returned error'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because pre-installation maintainer script failed'
HELPTEXT='
This is a somewhat unclassified failure at the moment.
'
piuparts-0.56/known_problems/pyshared_file_not_bytecompiled_inadequate_issue.conf 0000664 0000000 0000000 00000000710 12175701021 025751 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* pyshared-file-not-bytecompiled'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'pyshared-file-not-bytecompiled' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'pyshared-file-not-bytecompiled' which indicates a bug.
"
piuparts-0.56/known_problems/immediate_configuration_error.conf 0000664 0000000 0000000 00000000557 12253565512 022224 0 ustar #
# detect packages which failed because apt could not perform immediate configuration
#
PATTERN='E: Could not perform immediate configuration on'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because apt could not perform immediate configuration'
HELPTEXT='
This is a bug in apt, but it has to be worked around in some packages.
'
piuparts-0.56/known_problems/modified_files_after_purge_error.conf 0000664 0000000 0000000 00000000423 12145710343 022646 0 ustar #
# detect packages with modified files after purge
#
PATTERN='FAIL: After purging files have been modified:'
EXCLUDE_PATTERN=',|usr/local'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with modified files after purge'
HELPTEXT='
This is obviously broken.
'
piuparts-0.56/known_problems/conffile_prompt_error.conf 0000664 0000000 0000000 00000002316 12145710343 020512 0 ustar #
# detect packages with failed because they prompt due to a modified conffile
#
PATTERN='EOF on stdin at conffile prompt'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs with the string "EOF on stdin at conffile prompt"'
HELPTEXT='
The piuparts upgrade test failed because dpkg detected a conffile as being modified and then prompted the user for an action. As there is no user input, this fails. But this is not the real problem, the real problem is that this prompt shows up in the first place, as there was nobody modifying this conffile at all, the package has just been installed and upgraded...
This is explained in detail in policy 10.7.3 at http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.3 which says "[These scripts handling conffiles] must not ask unnecessary questions (particularly during upgrades), and must otherwise be good citizens."
According to the thread started at 200908191215.05079.holger@layer-acht.org these bugs are to be filed with severity serious .
'
piuparts-0.56/known_problems/disappeared_files_after_purge_error.conf 0000664 0000000 0000000 00000000435 12145710343 023352 0 ustar #
# detect packages with disappeared files after purge
#
PATTERN='FAIL: After purging files have disappeared:'
EXCLUDE_PATTERN=',|usr/local'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with files which disappeared after purge'
HELPTEXT='
This is obviously broken.
'
piuparts-0.56/known_problems/unowned_files_after_purge_issue.conf 0000664 0000000 0000000 00000001435 12145710343 022550 0 ustar #
# detect packages with unowned files after purge (policy 6.8)
#
PATTERN='not owned'
EXCLUDE_PATTERN="/usr/share/mime/|usr/local/"
WHERE='pass'
ISSUE=1
HEADER='Packages with unowned files after purge'
HELPTEXT='
WARNING: this list might contain false positives.
One group of them are packages with files in /usr/share/mime
- those are bugs from shared-mime-info (#527063 ) and have been ignored for this list.
There are probably others like this as well.
Packages with unowned files after purge (violating policy 6.8) see
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
'
piuparts-0.56/known_problems/cron_error_after_removal_error.conf 0000664 0000000 0000000 00000001457 12145710343 022411 0 ustar #
# detect packages with cron errors after the package has been removed
#
PATTERN='(FAIL: Cron file .* has output with package removed|ERROR: Command failed.*./etc/cron\..*/.*.])'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because a cron file has output and/or exits with error after the package has been removed'
HELPTEXT='
From http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.5
The scripts or crontab entries in these directories should check if all
necessary programs are installed before they try to execute them. Otherwise,
problems will arise when a package was removed but not purged since
configuration files are kept on the system in this situation.
'
piuparts-0.56/known_problems/processes_running_error.conf 0000664 0000000 0000000 00000002051 12145710343 021066 0 ustar #
# detect packages which failed because of processes are running inside chroot
#
PATTERN='ERROR: FAIL: Processes are running inside chroot'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because of processes left behind'
HELPTEXT='
There were processes running inside the chroot at the end of the piuparts run.
This is probably due to directly calling /etc/rc.d/ scripts in packages maintainer scripts, which is a violation of policy 9.3.3.2 and must be replaced by using invoke-rc.d (which will respect an optionally existing policy-rc.d) - see
http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.3.3 , /usr/share/doc/sysv-rc/README.invoke-rc.d.gz and /usr/share/doc/sysv-rc/README.policy-rc.d.gz.
According to the thread started at 200908061127.35727.holger@layer-acht.org these bugs are to be filed with severity serious .
'
piuparts-0.56/known_problems/problems_and_no_force_error.conf 0000664 0000000 0000000 00000001232 12145710343 021637 0 ustar #
# detect packages with problems because of not enough force
#
PATTERN='E: There are problems and -y was used without --force-yes'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because of not enough force'
HELPTEXT='
The piuparts logfile for these packages contains the string
E: There are problems and -y was used without --force-yes , which is usually an indication, that an essential package needs to be removed to install this package. As piuparts does not use that much force, the piuparts test fails.
This is usually not an error in the package and it needs to be seen how piuparts should deal with it.
'
piuparts-0.56/known_problems/dependency_error.conf 0000664 0000000 0000000 00000001035 12145710343 017437 0 ustar #
# detect packages with unsatisfied dependencies
#
PATTERN='E: Broken packages|E: Unable to correct problems, you have held broken packages.|E: Error, pkgProblemResolver::Resolve generated breaks, this may be caused by held packages.'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because of unsatisfied dependencies'
HELPTEXT='
Usually this is caused by some unsatisfied (versioned) Depends/Conflicts/Replaces.
These packages will be automatically rescheduled for testing seven days after they failed.
'
piuparts-0.56/known_problems/command_not_found_issue.conf 0000664 0000000 0000000 00000001451 12145710343 021013 0 ustar #
# detect packages which passed the piuparts test but have the string "command not found" in their logs
#
PATTERN='command not found|: not found'
WHERE='pass'
ISSUE=1
HEADER='Packages which passed the piuparts test but have logs with the string "command not found"'
HELPTEXT='
From the third paragraph about the meaning of the depends field in
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps :
The Depends field should also be used if the postinst, prerm or postrm scripts
require the package to be present in order to run. __Note, however, that the
postrm cannot rely on any non-essential packages to be present during the
purge phase__.
'
piuparts-0.56/known_problems/unowned_files_after_purge_error.conf 0000664 0000000 0000000 00000001455 12145710343 022553 0 ustar #
# detect packages with unowned files after purge (policy 6.8)
#
PATTERN='not owned'
EXCLUDE_PATTERN='/usr/share/mime/|usr/local/'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with unowned files after purge'
HELPTEXT='
WARNING: this list might contain false positives.
One group of them are packages with files in /usr/share/mime
- those are bugs from shared-mime-info (#527063 ) and have been ignored for this list.
There are probably others like this as well.
Packages with unowned files after purge (violating policy 6.8) see
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
'
piuparts-0.56/known_problems/incompatible_licenses_inadequate_issue.conf 0000664 0000000 0000000 00000000655 12175701021 024056 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* incompatible-licenses'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'incompatible-licenses' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'incompatible-licenses' which indicates a bug.
"
piuparts-0.56/known_problems/alternatives_after_purge_issue.conf 0000664 0000000 0000000 00000001232 12145710343 022403 0 ustar #
# detect packages with unowned files in /etc/alternatives after purge (policy 6.8)
#
PATTERN='/etc/alternatives/.*not owned'
WHERE='pass'
ISSUE=1
HEADER='Packages with leftover alternatives after purge'
HELPTEXT='
Packages with unowned files in /etc/alternatives after purge (violating policy 6.8) see
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
Alternatives are usually registered with update-alternatives in postinst configure and need to be unregistered again in prerm remove .
'
piuparts-0.56/known_problems/post_removal_script_error.conf 0000664 0000000 0000000 00000000517 12145710343 021423 0 ustar #
# detect packages which failed because post-removal maintainer script failed
#
PATTERN='subprocess .*post-removal script returned error'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because post-removal maintainer script failed'
HELPTEXT='
This is a somewhat unclassified failure at the moment.
'
piuparts-0.56/known_problems/inadequate_exit_issue.conf 0000664 0000000 0000000 00000000744 12155061511 020474 0 ustar #
# detect packages which have the string "Exit code from adequate was" in their logs
#
PATTERN='(WARN|FAIL): Exit code from adequate was'
WHERE='pass fail bugged affected'
ISSUE=0
HEADER='Packages which have logs with the string "Exit code from adequate was"'
HELPTEXT='
Running adequate resulted in an exit code not equal zero, which indicates a severe problem with adequate. Please investigate and report.
'
piuparts-0.56/known_problems/missing_md5sums_error.conf 0000664 0000000 0000000 00000000407 12155061511 020446 0 ustar #
# detect packages that ship files without md5sums
#
PATTERN='(FILE WITHOUT MD5SUM|MD5SUM FILE NOT FOUND)'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages containing files without md5sum'
HELPTEXT='
Packages that ship files that have no md5sum.
'
piuparts-0.56/known_problems/needs_rebuild_issue.conf 0000664 0000000 0000000 00000001214 12145710343 020123 0 ustar #
# detect packages which have the string "Please rebuild the package" or "package ... should be rebuilt" in their logs
#
PATTERN="Please rebuild the package|should be rebuilt|should be rebuild|warning: maintainer scripts should not call install-info anymore"
WHERE='pass'
ISSUE=1
HEADER='Packages which have logs with the string "Please rebuild the package" or "package ... should be rebuilt"'
HELPTEXT='
This is a recommendation to rebuild some packages with updated debhelper to enable new features, e.g. trigger support.
Please identify the correct package causing this warning and retest the rdepends after that package was fixed.
'
piuparts-0.56/known_problems/boring_obsolete_conffile_inadequate_issue.conf 0000664 0000000 0000000 00000000764 12175701021 024545 0 ustar #
# detect packages which have the string "Running adequate resulted in less interesting tags found" in their logs
#
PATTERN='(FAIL|WARN): Running adequate resulted in less interesting tags found: .* obsolete-conffile'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'obsolete-conffile' by adequate"
HELPTEXT="
Some issues detected by adequate are also detected by piuparts, 'obsolete-conffile' is one of them.
"
piuparts-0.56/known_problems/logrotate_error_after_removal_error.conf 0000664 0000000 0000000 00000001071 12145710343 023440 0 ustar #
# detect packages with logrotate errors after the package has been removed
#
PATTERN='(FAIL: Logrotate file .* has output with package removed|ERROR: Command failed.*'/etc/logrotate\.d/.*'])'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because a logrotate script has output and/or exits with error after the package has been removed'
HELPTEXT='
Most of these packages are probably not buggy but rather affected by #582630 . It is being considered to disable this check...
'
piuparts-0.56/known_problems/owned_files_after_purge_error.conf 0000664 0000000 0000000 00000001141 12145710343 022200 0 ustar #
# detect packages with owned files after purge (policy 6.8)
#
PATTERN='owned by:'
EXCLUDE_PATTERN=',|usr/local/'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with owned files after purge'
HELPTEXT='
WARNING: this list might contain false positives.
#316521: dpkg: incomplete cleanup of empty directories
Packages need to remove owned files after purge, see
http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.3
'
piuparts-0.56/known_problems/command_not_found_error.conf 0000664 0000000 0000000 00000001720 12145710343 021013 0 ustar #
# detect packages with miss a depends or use non-essential in purge
#
PATTERN='command not found|: not found'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs with the string "command not found"'
HELPTEXT='
From the third paragraph about the meaning of the depends field in
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-binarydeps
The Depends field should also be used if the postinst, prerm or postrm scripts
require the package to be present in order to run. __Note, however, that the
postrm cannot rely on any non-essential packages to be present during the
purge phase__.
NOTE: it has not been verified that this error really caused the package to
fail the piuparts test, but it did fail.
There are also successful logs with "command not found" (though not listed below).
'
piuparts-0.56/known_problems/resource_violation_error.conf 0000664 0000000 0000000 00000000677 12145710343 021247 0 ustar #
# detect packages violating resource limits during the test
#
PATTERN='Command was terminated after exceeding|Process KILLED - exceed maximum run time'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages where the test was aborted after exceeding time or output limit'
HELPTEXT='
Usually caused by seriously misbehaving maintainer scripts that go into endless loops or try to get user input in DEBIAN_FRONTEND=noninteractive mode.
'
piuparts-0.56/known_problems/packages_have_been_kept_back_error.conf 0000664 0000000 0000000 00000000515 12145710343 023100 0 ustar #
# detect possible dependency issues
#
PATTERN='packages have been kept back'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with logs with the string "packages have been kept back"'
HELPTEXT='
"packages have been kept back" usually indicates some dependency issue
that caused apt to not fully upgrade the system.
'
piuparts-0.56/known_problems/unknown_purge_error.conf 0000664 0000000 0000000 00000001074 12145710343 020225 0 ustar #
# detect packages which try to overwrite other packages files
#
PATTERN='ERROR: Command failed .*.dpkg., .--purge., .--pending.]'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because dpkg --purge --pending failed'
HELPTEXT='
This is often because some dependencies have to be removed together and not seperatedly, and thus rather a bug in piuparts... but be careful, this list also includes failures due to "command not found"-error , which made the purge fail... so file those bugs first.
'
piuparts-0.56/known_problems/owned_files_by_many_packages_error.conf 0000664 0000000 0000000 00000001134 12145710343 023173 0 ustar #
# detect conffiles owned by two or more packages which are not removed by dpkg correctly
#
PATTERN='owned by.+,'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs which have conffiles owned by two or more packages which are not removed by dpkg correctly'
HELPTEXT='
So these might be ok packages (or not, as there might be other problems), but we cannot know until #454694 in dpkg is fixed.
#316521: dpkg: incomplete cleanup of empty directories
'
piuparts-0.56/known_problems/insserv_error.conf 0000664 0000000 0000000 00000002354 12145710343 017017 0 ustar #
# detect packages with fail due to insserv error
#
PATTERN='insserv: exiting now'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs with the string "insserv: exiting now"'
HELPTEXT='
Some packages fail to install because their init.d scripts have
headers with bugs. There are several classes of bugs.
Some packages
fail to install because the init.d script have conflicting
provide. This is normally reported like this:
insserv: script clvm: service lvm already provided!
insserv: exiting now!
Other packages fail to install because their dependencies are missing.
This is normally reported like this:
insserv: Service portmap has to be enabled to start service quotarpc
insserv: exiting now!
Last, some packages introduce dependency loops, this is normally
reported like this:
insserv: There is a loop between service script1 and script2 if started
insserv: exiting without changing boot order!
See the paragraph "How to solve migration problems" at http://wiki.debian.org/LSBInitScripts/DependencyBasedBoot for information how to fix these issues.
'
piuparts-0.56/known_problems/overwrite_other_packages_files_error.conf 0000664 0000000 0000000 00000001451 12145710343 023572 0 ustar #
# detect packages which try to overwrite other packages files
#
PATTERN='trying to overwrite (.*) which is also in package'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because they tried to overwrite other packages files'
HELPTEXT='
This is because the package tries to overwrite another packages files without declaring a replaces relation. See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces .
According to the thread started at 200908071233.02813.holger@layer-acht.org these bugs are to be filed with severity serious .
'
piuparts-0.56/known_problems/used_exception_issue.conf 0000664 0000000 0000000 00000000406 12253565512 020345 0 ustar #
# report packages that used a piuparts exception to pass a test
#
PATTERN='piuparts exception for package'
WHERE='pass'
ISSUE=1
HEADER='Packages that used a piuparts exception'
HELPTEXT='
These packages needed a piuparts exception to pass the test.
'
piuparts-0.56/known_problems/packages_have_been_kept_back_issue.conf 0000664 0000000 0000000 00000000475 12145710343 023104 0 ustar #
# detect possible dependency issues
#
PATTERN='packages have been kept back'
WHERE='pass'
ISSUE=1
HEADER='Packages with logs with the string "packages have been kept back"'
HELPTEXT='
"packages have been kept back" usually indicates some dependency issue
that caused apt to not fully upgrade the system.
'
piuparts-0.56/known_problems/undefined_symbol_inadequate_issue.conf 0000664 0000000 0000000 00000000636 12175701021 023050 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* undefined-symbol'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'undefined-symbol' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'undefined-symbol' which indicates a bug.
"
piuparts-0.56/known_problems/broken_symlinks_error.conf 0000664 0000000 0000000 00000001035 12145710343 020532 0 ustar #
# detect packages which have the string "Broken symlinks" in their logs
#
PATTERN='(WARN|FAIL): Broken symlink'
WHERE='fail bugged affected'
ISSUE=1
HEADER='Packages which have logs with the string "Broken symlinks"'
HELPTEXT='
This is clearly an error, but as there are too many of this kind, piuparts can be configured to not fail if it detects broken symlinks. Another option is not to test for broken symlinks. See the piuparts manpage for details.
'
piuparts-0.56/known_problems/diversion_error.conf 0000664 0000000 0000000 00000000516 12145710343 017326 0 ustar #
# detect packages which mishandle diversions
#
PATTERN='ERROR: FAIL: (Existing|Installed) diversions'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages that leave a modified diversion state'
HELPTEXT='
Packages need to remove all the diversions they introduced and may not modify diversions setup by other packages.
'
piuparts-0.56/known_problems/boring_broken_symlink_inadequate_issue.conf 0000664 0000000 0000000 00000000753 12175701021 024110 0 ustar #
# detect packages which have the string "Running adequate resulted in less interesting tags found" in their logs
#
PATTERN='(FAIL|WARN): Running adequate resulted in less interesting tags found: .* broken-symlink'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'broken-symlink' by adequate"
HELPTEXT="
Some issues detected by adequate are also detected by piuparts, 'broken-symlink' is one of them.
"
piuparts-0.56/known_problems/db_setup_error.conf 0000664 0000000 0000000 00000001444 12145710343 017132 0 ustar #
# detect packages with failed to install due to problems configuring the db - see #595652
#
PATTERN='(warning: database package not installed|dbconfig-common: .* configure: (aborted|noninteractive fail).|updating database schema for .*...command failed with code 0|psql: could not connect to server: No such file or directory|DBI connect.* failed: could not connect to server|pg_pconnect\(\): Unable to connect to PostgreSQL server|Unable to connect to MySQL server|unable to connect to mysql server)'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because installation failed because no database could be connected.'
HELPTEXT='
Just how buggy these packages really are is discussed in #595652 at the moment.
'
piuparts-0.56/known_problems/debsums_mismatch_error.conf 0000664 0000000 0000000 00000000504 12145710343 020650 0 ustar #
# detect packages which modify shipped files
#
PATTERN='FAIL: debsums reports modifications inside the chroot'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with modified files before removal'
HELPTEXT='
Modifying conffiles is forbidden by Policy 10.7.3.
Modifying other shipped files is a stupid idea.
'
piuparts-0.56/known_problems/pre_depends_error.conf 0000664 0000000 0000000 00000001427 12145710343 017616 0 ustar #
# detect packages which failed because of a problem with pre-depends
#
PATTERN='E: Couldn.t configure pre-depend .* for .*, probably a dependency cycle.'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with failed logs because of a problem with pre-depends '
HELPTEXT='
The package(s) in question fail(s) to install or upgrade properly, because a pre-dependent package could not be configured. This is likely due to a dependency cycle.
Note that it is possible, that aptitude can deal with (some of) such situations (ie upgrades), (by removing the packages first), while apt-get cannot. While it can be argued this is a problem in apt-get, it is also a problem in the package(s) listed below, as both aptitude and apt-get can be used for installing packages.
'
piuparts-0.56/known_problems/alternatives_after_purge_error.conf 0000664 0000000 0000000 00000001252 12145710343 022406 0 ustar #
# detect packages with unowned files in /etc/alternatives after purge (policy 6.8)
#
PATTERN='/etc/alternatives/.*not owned'
WHERE='fail bugged affected'
ISSUE=0
HEADER='Packages with leftover alternatives after purge'
HELPTEXT='
Packages with unowned files in /etc/alternatives after purge (violating policy 6.8) see
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
Alternatives are usually registered with update-alternatives in postinst configure and need to be unregistered again in prerm remove .
'
piuparts-0.56/known_problems/debsums_mismatch_issue.conf 0000664 0000000 0000000 00000000464 12145710343 020654 0 ustar #
# detect packages which modify shipped files
#
PATTERN='FAIL: debsums reports modifications inside the chroot'
WHERE='pass'
ISSUE=1
HEADER='Packages with modified files before removal'
HELPTEXT='
Modifying conffiles is forbidden by Policy 10.7.3.
Modifying other shipped files is a stupid idea.
'
piuparts-0.56/known_problems/missing_copyright_file_inadequate_issue.conf 0000664 0000000 0000000 00000000701 12253565512 024265 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .*missing-copyright-file|MISSING COPYRIGHT FILE'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'missing-copyright-file' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'missing-copyright-file' which indicates a bug.
"
piuparts-0.56/known_problems/installs_over_symlink_issue.conf 0000664 0000000 0000000 00000001132 12145710343 021750 0 ustar #
# detect packages which have the string "dirname part contains a symlink" in their logs
#
PATTERN='silently overwrites files via directory symlinks|dirname part contains a symlink'
WHERE='pass'
ISSUE=1
HEADER='Packages that install something over existing symlinks'
HELPTEXT='
Installing anything over a symlink opens a can of worms -
this causes problems on upgrades while switching between directory and symlink
or if the symlink is ever changed.
Piuparts looks at all $pathname known to dpkg and checks for
$(dirname $pathname) != $(readlink $(dirname $pathname))
'
piuparts-0.56/known_problems/symbol-size-mismatch_inadequate_issue.conf 0000664 0000000 0000000 00000000652 12253565512 023612 0 ustar #
# detect packages with some inadequate tag from adequate
#
PATTERN='(FAIL|WARN): Running adequate resulted in .* symbol-size-mismatch'
WHERE='pass fail bugged affected'
ISSUE=1
HEADER="Packages tagged 'symbol-size-mismatch' by adequate"
HELPTEXT="
Running adequate resulted in the package being tagged 'symbol-size-mismatch' which indicates a bug.
"
piuparts-0.56/update-piuparts-master-setup 0000775 0000000 0000000 00000005252 12145710343 015713 0 ustar #!/bin/sh
set -e
#
# update piuparts master setup from git (eg. used on pejacevic.debian.org)
#
# Copyright 2009-2013 Holger Levsen (holger@layer-acht.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
if [ "`id -n -u`" != "piupartsm" ] ; then
echo please run this script as piupartsm user
exit 1
fi
PIUPARTS_PREFIX=/srv/piuparts.debian.org
PIUPARTS_HTDOCS=$PIUPARTS_PREFIX/htdocs
PIUPARTS_TMPDIR=$PIUPARTS_PREFIX/tmp
#
# create $PIUPARTS_PREFIX
#
if [ ! -d $PIUPARTS_PREFIX ] ; then
sudo mkdir -p $PIUPARTS_PREFIX
sudo chown piupartsm:piuparts $PIUPARTS_PREFIX
sudo chmod 0775 $PIUPARTS_PREFIX
fi
#
# update source
#
if [ ! -d $PIUPARTS_PREFIX/src ] ; then
mkdir -p $PIUPARTS_PREFIX/src
chmod 0755 $PIUPARTS_PREFIX/src
cd $PIUPARTS_PREFIX/src
git clone git://git.debian.org/git/piuparts/piuparts.git
cd piuparts
git checkout develop
fi
cd $PIUPARTS_PREFIX/src/piuparts
pwd
# git checkout branch if $1 is given
if [ ! -z "$1" ] ; then
git checkout $1
fi
# git fetch+pull if $2 is given
if [ ! -z "$2" ] ; then
git fetch $2
git pull $2 $1
fi
#
# install everything from GIT into PIUPARTS_PREFIX
#
make clean
make prefix=$PIUPARTS_PREFIX \
build build-doc
make prefix=$PIUPARTS_PREFIX \
docdir=$PIUPARTS_HTDOCS/doc \
htdocsdir=$PIUPARTS_HTDOCS \
install install-doc install-conf-4-running-from-git
make clean
#
# install copies of the weather icons
# to avoid needing FollowSymlinks in the apache config
#
for icon in weather-severe-alert.png sunny.png
do
if [ -e /usr/share/icons/Tango/24x24/status/$icon ] ; then
cp -f /usr/share/icons/Tango/24x24/status/$icon $PIUPARTS_HTDOCS/images/$icon
fi
done
#
# update $PIUPARTS_PREFIX
#
cd $PIUPARTS_PREFIX
pwd
mkdir -p master backup
# to support multiple hosts with this setup
cd etc/piuparts
HOSTNAME=$(hostname)
for f in piuparts.conf
do
ln -sf $f.$HOSTNAME $f
done
#
# create working dir
#
mkdir -p $PIUPARTS_TMPDIR
#
# update master home
#
cd
pwd
ln -sf $PIUPARTS_PREFIX/share/piuparts/master bin
crontab $PIUPARTS_PREFIX/etc/piuparts/crontab-master
echo "Update finished."
piuparts-0.56/slave-bin/ 0000775 0000000 0000000 00000000000 12253565401 012066 5 ustar piuparts-0.56/slave-bin/slave_run.in 0000775 0000000 0000000 00000004405 12145710343 014417 0 ustar #!/bin/sh
set -e
# Copyright 2009-2013 Holger Levsen (holger@layer-acht.org)
# Copyright © 2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Run (several) piuparts-slave instance(s) in screen to allow interactive
# control later on.
#
. @sharedir@/piuparts/lib/read_config.sh
get_config_value PYTHONPATH global PYTHONPATH ''
get_config_value SLAVEROOT global slave-directory
get_config_value PIUPARTS_TMPDIR global tmpdir
get_config_value SLAVECOUNT global slave-count 1
export PYTHONPATH
SESSIONNAME=piuparts_slave_screen
SCREENLOG=$SLAVEROOT/screenlog.0
if ! screen -ls $SESSIONNAME | grep -q "No Sockets found" ; then
echo "piuparts-slave is already running!"
echo
screen -ls
exit 1
fi
# cleanup cruft from previous runs
@sharedir@/piuparts/slave/slave_cleanup
rm -f $SCREENLOG
# ensure the temporary directory exists
mkdir -p $PIUPARTS_TMPDIR
mkdir -p $SLAVEROOT
cd $SLAVEROOT
# Ensure the screen session exists, run normal shell in screen 0
screen -S $SESSIONNAME -d -m
echo "Started screen session '$SESSIONNAME'."
# run this on a single slave or a sequence of slaves
# FIXME: this should really test whether $1 is an integer and within SLAVECOUNT
if [ "$1" != "" ] ; then
SLAVES=$1
else
SLAVES="$(seq $SLAVECOUNT)"
fi
for SLAVENUM in $SLAVES
do
mkdir -p $SLAVENUM
SLAVEDIR=$(readlink -f $SLAVENUM)
# Ensure there is a window for this slave.
screen -S $SESSIONNAME -X screen -t slave$SLAVENUM -L $SLAVENUM
# Launch the slave.
screen -S $SESSIONNAME -p $SLAVENUM -X stuff "
cd $SLAVEDIR
@sharedir@/piuparts/piuparts-slave
"
echo "piuparts-slave $SLAVENUM has been started."
done
piuparts-0.56/slave-bin/detect_slave_problems.in 0000775 0000000 0000000 00000004217 12145710343 016767 0 ustar #!/bin/sh
set -e
# Copyright 2009-2010 Holger Levsen (holger@layer-acht.org)
# Copyright © 2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# this scripts monitors the output of piuparts-slave
# when running in screen started by ~piupartss/bin/slave_run
#
. @sharedir@/piuparts/lib/read_config.sh
# outputs file age in seconds (or 0 if the file does not exist)
file_age()
{
if [ -e "$1" ]; then
local ctime now
ctime=$(stat -c %Z "$1" 2>/dev/null || echo 0)
now=$(date +%s)
echo $(($now - $ctime))
else
echo "0"
fi
}
get_config_value SLAVEROOT global slave-directory
get_config_value IDLE_SLEEP global idle-sleep 1800
SCREENLOG=$SLAVEROOT/screenlog.0
STATEFILE=$SLAVEROOT/slave-problems
# clear the statefile daily and whine again
test $(file_age $STATEFILE) -lt 86000 || rm -f $STATEFILE
# Only complain if screenlog is older than $IDLE_SPEEP + 1 minute (the slave
# likes to sleep that long) and the problem is new or was not reported within
# the previous 24 hours.
if [ $(file_age $SCREENLOG) -le $(($IDLE_SLEEP + 60)) ]; then
rm -f $STATEFILE
elif [ ! -f $STATEFILE ]; then
{
echo "Either a test is running for a very long time (but no test"
echo "should run longer than an hour), piuparts-slave hangs or is"
echo "not running at all or wasn't started with"
echo "~piupartss/bin/slave_run - please investigate and take"
echo "appropriate measures!"
echo
tail $SCREENLOG
} | mail -s "problem with piuparts-slave detected" piupartss
touch $STATEFILE
fi
piuparts-0.56/slave-bin/detect_tmp_cruft.in 0000775 0000000 0000000 00000003267 12144653075 015770 0 ustar #!/bin/sh
set -e
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2012-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# check for stale mounts and chroots in $PIUPARTS_TMPDIR
#
. @sharedir@/piuparts/lib/read_config.sh
get_config_value PIUPARTS_TMPDIR global tmpdir
MOUNTS="$(mktemp)"
cp /proc/mounts "$MOUNTS"
if [ "$(grep -v trash "$MOUNTS" | grep -c "$PIUPARTS_TMPDIR")" -gt 1 ] ; then
echo "More than one mountpoint below $PIUPARTS_TMPDIR detected!"
echo
grep "$PIUPARTS_TMPDIR" "$MOUNTS"
echo
echo "Zero or one mountpoint is normal for piuparts operation, more is not."
echo "Please investigate and cleanup."
echo
fi
rm "$MOUNTS"
LS_TMP=$(ls --color=never -l "$PIUPARTS_TMPDIR")
if [ "$(echo "$LS_TMP" | wc -l)" -gt 12 ] ; then
echo "More than ten directories in $PIUPARTS_TMPDIR detected!"
echo
echo "$LS_TMP"
echo
echo "$(du -shx "$PIUPARTS_TMPDIR" 2>/dev/null)"
echo
echo "One is normal for piuparts operation, more is not."
echo "Please investigate and cleanup."
echo
fi
piuparts-0.56/slave-bin/slave_join 0000775 0000000 0000000 00000002407 12144653075 014154 0 ustar #!/bin/sh
set -e
# Copyright 2009-2010 Holger Levsen (holger@layer-acht.org)
# Copyright © 2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# reattach to an existing slave session
#
SESSIONNAME=piuparts_slave_screen
if screen -ls $SESSIONNAME | grep -q "No Sockets found" ; then
echo "piuparts-slave not running!"
echo
echo "ps faxu | grep piuparts"
ps faxu | grep piuparts
echo
echo "screen -ls"
screen -ls
exit 1
fi
if [ -w $(tty) ]; then
screen -x -S $SESSIONNAME
else
# use script hack to get a new writable tty
script -q -c "screen -x -S $SESSIONNAME" /dev/null
fi
piuparts-0.56/slave-bin/slave_cleanup.in 0000775 0000000 0000000 00000003132 12144653075 015245 0 ustar #!/bin/sh
set -e
# Copyright 2012 Holger Levsen (holger@layer-acht.org)
# Copyright © 2012-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cleanup old chroots etc. in $PIUPARTS_TMPDIR
#
# - this should only be run (automatically) on boot
#
. @sharedir@/piuparts/lib/read_config.sh
get_config_value PIUPARTS_TMPDIR global tmpdir
# do nothing if piuparts-slave is running
if pgrep -f share/piuparts/piuparts-slave >/dev/null ; then
exit 0
fi
# umount all mount points (should be none on boot, but this script can also be called at other times)
for MOUNTPOINT in $(tac /proc/mounts | cut -d " " -f 2 | grep "$PIUPARTS_TMPDIR/")
do
sudo umount "$MOUNTPOINT"
done
# cleanup tmp
mkdir -p "$PIUPARTS_TMPDIR"
for dir in $(ls -d1 "$PIUPARTS_TMPDIR"/*/ 2>/dev/null || true)
do
if sudo test -f "$dir/.piuparts.tmpdir"; then
sudo rm -rf --one-file-system "$dir"
test ! -d "$dir" || sudo touch "$dir/.piuparts.tmpdir"
fi
done
piuparts-0.56/slave-bin/detect_leftover_processes.in 0000775 0000000 0000000 00000003742 12144653075 017677 0 ustar #!/bin/sh
set -e
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2012-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# find processes running in deleted chroots
#
. @sharedir@/piuparts/lib/read_config.sh
# outputs file age in seconds (or 0 if the file does not exist)
file_age()
{
if [ -e "$1" ]; then
local ctime now
ctime=$(stat -c %Z "$1" 2>/dev/null || echo 0)
now=$(date +%s)
echo $(($now - $ctime))
else
echo "0"
fi
}
get_config_value SLAVEROOT global slave-directory
get_config_value PIUPARTS_TMPDIR global tmpdir
STATEFILE=$SLAVEROOT/leftover_processes
# clear the statefile daily and whine again
test $(file_age $STATEFILE) -lt 86000 || rm -f $STATEFILE
OUTPUT="$(sudo ls --color=never -lad /proc/*/root 2>/dev/null | grep "$PIUPARTS_TMPDIR" | grep "(deleted)")"
if [ -z "$OUTPUT" ]; then
rm -f $STATEFILE
elif [ "$(cat $STATEFILE 2>/dev/null)" != "$OUTPUT" ]; then
echo "Found processes running with a deleted chroot in $PIUPARTS_TMPDIR"
echo "This is usually because of 'FAIL: Processes are running inside chroot' which"
echo "usually means the package violates 'must use invoke-rc.d (policy 9.3.3.2)'."
echo
echo "$OUTPUT"
echo
echo "Please cleanup manually."
echo "Since #522918 has been fixed this should no longer happen."
echo "$OUTPUT" > $STATEFILE
fi
piuparts-0.56/NEWS 0000664 0000000 0000000 00000014655 12144650665 010726 0 ustar NEWS file for piuparts
======================
piuparts is a tool for testing that .deb packages can be
installed, upgraded, and removed without trouble. See
the README file and the manual page for more
information. This file summarizes the major changes,
particularly user visible changes, for each release.
Detailed change information can be found in bzr commit
messages.
This file is _deprecated_ now, see debian/NEWS.Debian instead!
Version 0.20, September 22, 2006
--------------------------------
When running external commands, use subprocess.Popen so that
there is no need for tricky (and therefore buggy) quoting
of shell command line arguments.
Version 0.19, September 8, 2006
-------------------------------
When reporting a bad symlink, show the target.
Version 0.18, September 7, 2006
-------------------------------
New features
Piuparts now checks for symlinks whose target does not
exist.
Option parsing code has been rewritten, and --help now
works better.
The chroot is now minimized before used: all unnecessary
packages are purged.
/dev/MAKEDEV, /etc/nologin, /usr/doc/cpio,
/var/spool/cron added to default ignores.
A version number may now begin with a + character. There
was a package that did that and piuparts crashed.
Version 0.17
------------
Bug fixes
The configuration files of piuparts-master/slave are
now documented in the README.
The Python profiler is no longer used. It used to be,
but that was a leftover from development (also known
as failure to read diffs before committing).
When testing upgrades between distributions, piuparts
now makes sure that the packages being tested are upgraded,
even if it means removing an old version of a dependency.
New features
Piuparts now checks for processes running inside the
chroot after it has installed or purged a package.
Because it uses policy-rc.d to prevent any services
from starting, no processes should run inside the
chroot after an installation or a purge has completed.
This check then finds packages that don't use invoke-rc.d
to start new processes.
A number of new default ignores have been added:
/etc/modprobe.d, compiled versions of debconf's Python
modules, papercut, ssl certificates.
/proc is now mounted (and unmounted) inside the chroot.
Version 0.16
------------
Bug fixes
The temporary directory for the chroot was not being
removed in all cases when running apt-get failed. This
has now been fixed.
New features
Added piuparts-analyze.py, which looks at new logs of
failed tests and compares them to logs of failed tests
for earlier versions of the same packages, and if so,
moves them automatically around. This saves a bit of
manual works. Thanks to Goswin Brederlow for the idea.
When piuparts creates a chroot from a tarball (option
-b), it now upgrades it before using it.
A number of new entries to the default ignores list.
Log files are now created with permissions that are
0666 modified with the process umask.
piuparts-report.py has been optimized somewhat.
Version 0.15
------------
Bug fixes
The dependency parser in piupartslib now understands
< and > (they're deprecated but one or two packges still
use them). It also now allows underscores in package
names because of the type-handling package.
Small fixes to the manual page.
New features and significant user visible changes
piuparts-master now understands Provides: headers.
A number of new entries to the default ignores list.
New option --keep-sources-list from John Wright.
Version 0.14
------------
Bug fixes
Specifications for ignoring some directories were buggy
and have now been fixed: /var/spool/news, /var/lib/cvs.
When testing a .deb file given on the command line,
if any of its dependencies were missing, the package
itself would be removed instead of piuparts reporting
an error. This has been fixed.
The check for whether a package is marked untestable
for piuparts-master now works.
New features and significant user visible changes
New program piuparts-report.py produces some "statistics"
about packages and their status with regard to testing
with piuparts-slave.
The chroot is always set up for piuparts, even if it is
unpacked from a tarball. This reduces problems with
out-of-date chroots and with using the pbuilder base.tgz
tarball.
Now ignored by default: /var/lib/firebird, /var/spool/news,
/var/lib/rbldns, /home/ftp.
Version 0.13
------------
Bug fixes
The configuration for apt-get (in the chroot) to allow
un-authenticated sources now actually works. There used
to be a typo.
New features and other user visible changes
The old run-piuparts.py script has been replaced by a
distributed system, consisting of piuparts-master.py and
piuparts-slave.py, plus a reporting script
piuparts-report.py. Since these are not useful for most
users, they're not installed on $PATH, but in
/usr/share/piuparts instead.
The slave part also runs upgrade tests between Debian
releases, which run-piuparts.py didn't.
Some additional files are ignored by default when
comparing the state of the chroot before and after
package installation.
piuparts-0.56/piuparts.1.txt 0000664 0000000 0000000 00000041371 12155061511 012755 0 ustar piuparts(1)
===========
:doctype: manpage
NAME
----
piuparts - .deb installation, upgrade, and removal testing suite
SYNOPSIS
--------
*piuparts* ['-apvV'] ['-d' _distro_] ['-i' _filename_] ['-I' _regexp_] ['-l' _logfile_] ['-m' _url_] ['--bindmount' _dir_] [_package_]... [_changes_file_]...
DESCRIPTION
-----------
*piuparts* tests that Debian packages handle installation, upgrading, and removal correctly. It does this by creating a minimal Debian installation in a chroot, and installing, upgrading, and removing packages in that environment, and comparing the state of the directory tree before and after. *piuparts* reports any files that have been added, removed, or modified during this process.
*piuparts* is meant as a quality assurance tool for people who create Debian packages to test them before they upload them to the Debian package
archive.
By default, piuparts can do three different tests:
. A simple install-purge test within one Debian distribution (chosen with the '-d' option, unstable by default). It sets up the chroot with the desired distribution, then installs and purges the packages, and reports problems.
. A simple install-upgrade-purge test within one Debian distribution. This test is like the install-purge test, but install the packages first via *apt-get* and then from the package files given on the command line. If the command line has package names (option '--apt' used), or no tested package is known to *apt-get* (new packages), this test is skipped, otherwise it is performed automatically.
. An upgrade test between Debian releases. This test is enabled by using the '-d' option multiple times and disables the other two tests. It sets up the chroot with the first distribution named, then upgrades it to each successive one, and then remembers the directory tree state at the end. After this, it starts over with the chroot of the first distribution, installs the desired packages (via *apt-get*), and does the successive upgrading (via *apt-get dist-upgrade*). Then, if package files (and not just package names) were given on the command line, it installs them. Finally, it reports problems against the state of the directory tree at the last distribution compared with the state without the packages having been installed. This test can be quite slow to execute.
+
Note that this does not work with experimental, because *apt-get* does not automatically upgrade to packages in experimental. To test a particular package or group of packages in experimental, use the second test.
Command line arguments are the paths to package files (e.g., _piuparts_1.0-1_all.deb_), paths to changes files (e.g., _piuparts_1.0-1_i386.changes_), or names of packages, if the '--apt' option is given.
When processing changes files, by default, all packages in a changes file will be processed together with all individual packages given on the command line. Then each package given on the command line is processed in a single group. If the '--single-changes-list' is used, the packages in all changes files are processed together along with any individual packages that were given on the command line.
*piuparts* outputs to the standard output some log messages to show what is going on. If a log file is used, the messages go there as well.
*piuparts* needs to be run as root.
OPTIONS
-------
Options must come before the other command line arguments.
*-a*, *--apt*::
The package arguments on the command line are to be treated as package names and installed via *apt-get* instead of being names of package files, to be installed via *dpkg -i*.
*--allow-database*::
Allow starting MySQL and PostgreSQL database servers in the chroot for
packages requiring database access in their maintainer scripts.
Do not use this option if there is already a database server running on the
system running piuparts (or piuparts-slave)!
In master-slave setups with multiple slaves running on one host collisions
may occur, these will be detected by 'detect_piuparts_issues' and the
affected packages will be tested again.
*--arch*='arch'::
Create chroot and run tests for (non-default) architecture 'arch'. The default is the output from 'dpkg --print-architecture'.
*-b* 'tarball', *--basetgz*='tarball'::
Use tarball as the contents of the initial chroot, instead of building a new one with debootstrap.
+
The tarball can be created with the '-s' option, or you can use one that *pbuilder* has created (see '-p'). If you create one manually, make sure the root of the chroot is the root of the tarball.
*--bindmount*='dir'::
Bind-mount a directory inside the chroot.
*-d* 'name', *--distribution*='name'::
Which Debian distribution to use: a code name (for example squeeze, wheezy or sid) or experimental. The default is sid (=unstable).
*-D* 'flavor', *--defaults*='flavor'::
Use default settings suitable for a particular flavor of Debian: either debian or ubuntu. The default is debian.
*--do-not-verify-signatures*::
Do not verify signatures from the Release files when running debootstrap.
*--dpkg-force-confdef*::
Make dpkg use --force-confdef, which lets dpkg always choose the default action when a modified conffile is found. This option will make piuparts ignore errors it was designed to report and therefore should only be used to hide problems in depending packages. 'This option shall normally not be used.' (See #466118.)
*--dpkg-noforce-unsafe-io*::
Prevent running dpkg with --force-unsafe-io. --force-unsafe-io causes dpkg to skip certain file system syncs known to cause substantial performance degradation on some filesystems. Thus, including this option reverts to safe but slower behavior. The --dpkg-noforce-unsafe-io is required for running tests on distributions older than squeeze.
*--no-eatmydata*::
Prevent use of eatmydata. The --no-eatmydata option is required for running tests on squeeze or older distributions.
*--extra-old-packages*='pkg1[,pkg2]...'::
Install additional old packages before upgrading. Allows testing package renames/merges where the old package is no longer available in the new distribution and the new one utilizes Conflicts/Replaces. The argument is a comma separated list of package names and the option can be given multiple times.
For install/purge tests these packages will be installed before the package
that is to be tested.
*-e* 'dirname', *--existing-chroot*='dirname'::
Use the specified directory as source for the new chroot, instead of building
a new one with debootstrap. This is similar to '--basetgz', but the contents
are not archived.
*--distupgrade-to-testdebs*::
Use the "testdebs" repository to override the packages in the distupgrade
target distribution. This allows to test complex upgrade paths before the
packages enter the archive.
*--extra-repo*='deb-line'::
Provide an additional (unparsed) line to be appended to sources.list, e.g.
'deb ' or
'deb file:// ./'
Useful for e.g. backports, security or local repositories that cannot be
handled by '--mirror'. May be repeated to add more than one line.
*-i* 'filename', *--ignore*='filename'::
Add a filename to the list of filenames to be ignored when comparing changes before and after installation. By default, piuparts ignores files that always change during a package installation and uninstallation, such as *dpkg* status files. The filename should be relative to the root of the chroot (e.g., _var/lib/dpkg/status_). This option can be used as many times as necessary.
*-I* 'regexp', *--ignore-regexp*='regexp'::
Add a regular expression pattern to the list of patterns for filenames to be ignored when comparing changes before and after installation. This option can be used as many times as necessary.
*--install-purge-install*::
Purge package after installation and reinstall. All depedencies are installed during purge.
*--install-recommends*::
Enable installation of Recommends.
*--install-remove-install*::
Remove package after installation and reinstall. For testing installation in config-files-remaining state.
*-k*, *--keep-tmpdir*::
Don't remove the temporary directory for the chroot when the program ends.
*-K*, *--keyring*='filename'::
Use FILE as the keyring to use with debootstrap when creating chroots.
*--keep-sources-list*::
Don't modify the chroot's etc/apt/sources.list (only makes sense with '--basetgz').
*--list-installed-files*::
List the files added to the chroot after the installation of the package and after the installation of the package dependencies.
*--lvm-volume*='lvm-volume'::
Use the specified lvm-volume as source for the chroot, instead of building a
new one with debootstrap. This creates a snapshot of the given LVM volume and
mounts it to the chroot path.
*--lvm-snapshot-size*='snapshot-size'::
Use the specified snapshot-size as snapshot size when creating a new LVM
snapshot (default: 1G)
*-l* 'filename', *--log-file*='filename'::
Append log file to _filename_ in addition to the standard output.
*--log-level*='level'::
Display messages from loglevel LEVEL, possible values are: error, info, dump, debug. The default is dump.
*-m* 'url', *--mirror*='url'::
Which Debian mirror to use. The default is the first mirror named in _/etc/apt/sources.list_ or _http://cdn.debian.net/debian_ if none is found. This option may be used multiple times to use multiple mirrors. Only the first mirror is used with *debootstrap*.
+
The 'components' that are used for a mirror can also be set with this option: a space separated list within the same argument (so you need to quote the entire argument in the shell). If no components are given explicitly, the usual Debian components are used (main, contrib, and non-free). For the mirrors read from _/etc/apt/sources.list_, the components are read from the same place.
+
Note that file: addresses works if the directories are made accessible from within the chroot with '--bindmount'.
*--no-adequate*::
Don't run adequate after installation. The default is to run adequate, provided it is installed.
*--no-diversions*::
Don't check for broken diversions.
*-n*, *--no-ignores*::
Forget all built-in and other ignores that have been set so far. Any '-i' or '-I' arguments that come after this one will be obeyed, but none of the ones that come before.
*-N*, *--no-symlinks*::
Don't check for broken symlinks.
*--fail-if-inadequate*::
Fail on inadequate results from running adequate. The default is to just issue those errors as warnings.
*--fail-on-broken-symlinks*::
Fail on broken symlinks. The default is to just issue those errors as warnings.
*--no-upgrade-test*::
Skip testing upgrade from an existing version in the archive.
*--no-install-purge-test*::
Skip the install and purge test.
*-p, *--pbuilder*::
Use _/var/cache/pbuilder/base.tgz_ as the base tarball. This is a shorthand so that you don't need to use '-b' for it.
*--pedantic-purge-test*::
Be pedantic when checking if a purged package leaves files behind. If this option is not set, files left in _/tmp_ are ignored.")
*--proxy*='URL'::
Use the proxy at URL to access the Debian mirror(s). Takes precedence over
the 'http_proxy' environment variable. Using a local proxy is recommended
because piuparts may use large amounts of bandwidth to repeatedly download
the same files.
*-s* 'filename', *--save*='filename'::
Save the chroot, after it has been set up, as a tarball into _filename_. It can then be used with '-b'.
*-B* 'FILE', *--end-meta*='FILE'::
Load chroot package selection and file meta data from FILE. See the function install_and_upgrade_between_distros() in piuparts.py for defaults. Mostly useful for large scale distro upgrade tests.
*-S* 'FILE', *--save-end-meta*='FILE'::
Save chroot package selection and file meta data in FILE for later use. See the function install_and_upgrade_between_distros() in piuparts.py for defaults. Mostly useful for large scale distro upgrade tests.
*--scriptsdir*='DIR'::
Directory where are placed the custom scripts. For more information about this, read README_server.txt
*--schroot*='SCHROOT-NAME'::
Use schroot session named SCHROOT-NAME for the chroot, instead of building a new one with debootstrap.
*--single-changes-list*::
When processing changes files, piuparts will process the packages in each individual changes file seperately. This option will set piuparts to scan the packages of all changes files together along with any individual package files that may have been given on the command line.
*--skip-minimize*::
Allow skip minimize chroot step. This is useful when you want to test several packages with piuparts. You can prepare a tarball already minimized and skip this step in all the tests. This is the default now.
*--minimize*::
Minimize the chroot with debfoster. This used to be the default until #539142 was fixed.
*--skip-cronfiles-test*::
Skip testing the output from the cron files left in the system after remove a package.
*--skip-logrotatefiles-test*::
Skip testing the output from the logrotate files left in the system after remove a package.
*--testdebs-repo*='deb-line'::
Provide an additional line to be appended to sources.list, e.g.
'deb ' or
'deb file:// ./'
If only an URL or local path is given as argument, "deb", "file://", and
"./" will be prepended/appended as needed.
The "testdebs" repository provides the packages to be tested (and some
additional dependencies, if needed, e.g. all packages built from the same
source package as the (binary) package being tested) and can be used for
testing complex installation and upgrade scenarios involving dependencies
that are not yet in the archive. This repository will be available only
for installing the target packages. Dependency resolution will be done by
apt-get. The packages to be tested can be passed as .debs or as package
names (with '--apt').
*-t directory*, *--tmpdir*='directory'::
Use directory as the place where temporary files and directories are created. The default is the environment variable *TMPDIR*, or _/tmp_ if not set.
+
Note: the temporary directory must *not* be mounted with the _nodev_ or _nosuid_ mount option.
*-v*, *--verbose*::
This option no longer has any meaning, but it is still accepted for backwards compatibility.
*-V*, *--version*::
Write out the version number of the program.
*--warn-on-debsums-errors*::
Print a warning rather than failing if debsums reports modified files.
*--warn-on-leftovers-after-purge*::
Print a warning rather than failing if files are left behind after purge.
*--warn-on-others*::
Print a warning rather than failing if files are left behind, modified, or removed by a package that was not given on the command-line.
+
This way, you can basically isolate the purge test to your own packages. If a package that is brought in as a dependency doesn't purge cleanly, the test will not fail because of it (but a warning message will be printed).
+
Behavior with multiple packages given on the command-line could be problematic, particularly if the dependency tree of one package in the list includes another in the list. Therefore, it is recommended to use this option with one package at a time.
EXAMPLES
--------
Assume that you have just built a new version of your Debian package, to be uploaded to Debian unstable. It is in _../foo_1.0-2_i386.deb_ and you would like to know whether it installs and uninstalls properly. Here's what you would do:
----
piuparts ../foo_1.0-2_i386.deb
----
If the package exists in the Debian archive already, the above command also tests that it upgrades properly.
To do the same test, but using a particular mirror, and only the main component, you would do this:
----
piuparts -m 'http://gytha/debian main' ../foo_1.0-2_i386.deb
----
If you want to do the same as above but for your changes files, pass in your changes files when running piuparts, and piuparts will process each package in the changes files as though you had passed all those packages on the command line to piuparts yourself. For example:
----
piuparts ../foo_1.0-2_i386.changes
piuparts -m 'http://gytha/debian main' ../foo_1.0-2_i386.changes
----
If you want to test that a package installs properly in the stable (currently squeeze) Debian release, then can be upgraded to the testing (currently wheezy) and unstable (sid) versions, and then uninstalled without problems, you would give the following command:
----
piuparts -a -d squeeze -d wheezy -d sid foo
----
ENVIRONMENT
-----------
*TMPDIR* Location for temporary files and directories. If not set, use _/tmp_. See also the '-t' ('--tmpdir') option.
NOTES
-----
Output of commands run by piuparts is limited to three megabytes. To change this limit, the source code needs to be edited.
Commands exceeding this limit will be aborted.
SEE ALSO
--------
*pbuilder*(1), *debootstrap*(8)
AUTHOR
------
Lars Wirzenius (liw@iki.fi) and others
DATE
----
2012-06-22
// vim: set filetype=asciidoc:
piuparts-0.56/master-bin/ 0000775 0000000 0000000 00000000000 12253565512 012252 5 ustar piuparts-0.56/master-bin/detect_well_known_errors.py 0000775 0000000 0000000 00000043233 12253565512 017737 0 ustar #!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2013 David Steele (dsteele@gmail.com)
#
# This file is part of Piuparts
#
# Piuparts is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# Piuparts is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, see .
import ConfigParser
import os
import time
import re
from collections import namedtuple
import argparse
import piupartslib
from piupartslib.conf import MissingSection
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
KPR_DIRS = ('pass', 'bugged', 'affected', 'fail')
KPR_EXT = '.kpr'
BUG_EXT = '.bug'
LOG_EXT = '.log'
TPL_EXT = '.tpl'
PROB_TPL = \
"""
$HEADER in $SECTION, sorted by reverse dependency count.
$HELPTEXT
The commandline to find these logs is:
COMMAND='$COMMAND'
Please file bugs!
Affected packages in $SECTION: $COUNT
"""
UNKNOWN_TPL = \
"""
Packages with unknown failures detected in $SECTION, sorted by reverse dependency count.
Please investigate and improve detection of known error types!
Please file bugs!
Affected packages in $SECTION: $COUNT
"""
PKG_ERROR_TPL = \
"""$RDEPS - $LOG
(PTS )
(BTS )
$BUG
"""
class WKE_Config(piupartslib.conf.Config):
"""Configuration parameters for Well Known Errors"""
def __init__(self):
self.section = 'global'
piupartslib.conf.Config.__init__(self, self.section,
{
"sections": "report",
"output-directory": "html",
"master-directory": ".",
"known-problem-directory": "@sharedir@/piuparts/known_problems",
"proxy": None,
}, "")
class WKE_Section_Config(piupartslib.conf.Config):
def __init__(self, section):
self.section = section
piupartslib.conf.Config.__init__(self, self.section,
{
"mirror": None,
"distro": None,
"area": None,
"arch": None,
"upgrade-test-distros": None,
}, "", defaults_section="global")
class Problem():
""" Encapsulate a particular known problem """
def __init__(self, probpath):
"""probpath is the path to the problem definition file"""
self.probpath = probpath
self.name = os.path.basename(probpath)
self.short_name = os.path.splitext(self.name)[0]
self.tags_are_valid = True
self.required_tags = ["PATTERN", "WHERE", "ISSUE",
"HEADER", "HELPTEXT"]
self.optional_tags = ["EXCLUDE_PATTERN", "EXPLAIN", "PRIORITY"]
self.init_problem()
for tag in self.required_tags:
if not tag in self.__dict__:
self.tags_are_valid = False
if "PATTERN" in self.__dict__:
self.inc_re = re.compile(self.PATTERN)
else:
self.inc_re = None
if "EXCLUDE_PATTERN" in self.__dict__:
self.exc_re = re.compile(self.EXCLUDE_PATTERN)
else:
self.exc_re = None
def valid(self):
return self.tags_are_valid
def init_problem(self):
"""Load problem file parameters (HELPTEXT="foo" -> self.HELPTEXT)"""
pb = open(self.probpath, 'r')
probbody = pb.read()
pb.close()
tagged = re.sub("^([A-Z]+=)", "\g<0>", probbody, 0, re.MULTILINE)
for chub in re.split('', tagged)[1:]:
(name, value) = re.split("=", chub, 1, re.MULTILINE)
while value[-1] == '\n':
value = value[:-1]
if re.search("^\'.+\'$", value, re.MULTILINE|re.DOTALL) \
or re.search('^\".+\"$', value, re.MULTILINE|re.DOTALL):
value = value[1:-1]
if name in self.required_tags or name in self.optional_tags:
self.__dict__[name] = value
else:
self.tags_are_valid = False
self.WHERE = self.WHERE.split(" ")
def has_problem(self, logbody, where):
"""Does the log text 'logbody' contain this known problem?"""
if where in self.WHERE:
if self.inc_re.search(logbody, re.MULTILINE):
for line in logbody.splitlines():
if self.inc_re.search(line):
if self.exc_re == None \
or not self.exc_re.search(line):
return True
return False
def get_command(self):
cmd = "grep -E \"%s\"" % self.PATTERN
if "EXCLUDE_PATTERN" in self.__dict__:
cmd += " | grep -v -E \"%s\"" % self.EXCLUDE_PATTERN
return cmd
class FailureManager():
"""Class to track known failures encountered, by package,
where (e.g. 'fail'), and known problem type"""
def __init__(self, logdict):
"""logdict is {pkgspec: fulllogpath} across all log files"""
self.logdict = logdict
self.failures = []
self.load_failures()
def load_failures(self):
"""Collect failures across all kpr files, as named tuples"""
for pkgspec in self.logdict:
logpath = self.logdict[pkgspec]
try:
kp = open(get_kpr_path(logpath), 'r')
for line in kp.readlines():
(where, problem) = self.parse_kpr_line(line)
self.failures.append(make_failure(where, problem, pkgspec))
kp.close()
except IOError:
print "Error processing %s" % get_kpr_path(logpath)
def parse_kpr_line(self, line):
"""Parse a line in a kpr file into where (e.g. 'pass') and problem name"""
m = re.search("^([a-z]+)/.+ (.+)$", line)
return (m.group(1), m.group(2))
def sort_by_path(self):
self.failures.sort(key=lambda x: self.logdict[x.pkgspec])
def sort_by_bugged_and_rdeps(self, pkgsdb):
self.pkgsdb = pkgsdb
def keyfunc(x, pkgsdb=self.pkgsdb, logdict=self.logdict):
pkg_obj = pkgsdb.get_package(get_pkg(x.pkgspec))
rdeps = pkgsdb.rrdep_count(pkg_obj)
is_failed = get_where(logdict[x.pkgspec]) == "fail"
return (not is_failed, -rdeps, logdict[x.pkgspec])
self.failures.sort(key=keyfunc)
def filtered(self, problem):
return [x for x in self.failures if problem==x.problem]
def make_failure(where, problem, pkgspec):
return (namedtuple('Failure', 'where problem pkgspec')(where, problem, pkgspec))
def get_where(logpath):
"""Convert a path to a log file to the 'where' component (e.g. 'pass')"""
return logpath.split('/')[-2]
def replace_ext(fpath, newext):
basename = os.path.splitext(os.path.split(fpath)[1])[0]
return '/'.join(fpath.split('/')[:-1] + [basename + newext])
def get_pkg(pkgspec):
return pkgspec.split('_')[0]
def get_kpr_path(logpath):
"""Return the kpr file path for a particular log path"""
return replace_ext(logpath, KPR_EXT)
def pts_subdir(source):
if source[:3] == "lib":
return source[:4]
else:
return source[:1]
def source_pkg(pkgspec, db):
source_name = db.get_control_header(get_pkg(pkgspec), "Source")
return source_name
def get_file_dict(workdirs, ext):
"""For files in [workdirs] with extension 'ext', create a dict of
_: """
filedict = {}
for dir in workdirs:
for fl in os.listdir(dir):
if os.path.splitext(fl)[1] == ext:
filedict[os.path.splitext(os.path.basename(fl))[0]] \
= os.path.join(dir, fl)
return filedict
def get_pkgspec(logpath):
"""For a log full file spec, return the pkgspec (_mtime(cachedict[pkgspec]) and not skipnewer)\
or get_where(logdict[pkgspec]) != get_where(cachedict[pkgspec])\
or recheck\
or (recheck_failed and not get_where(cachedict[pkgspec]) in ['pass']):
os.remove(cachedict[pkgspec])
count = count + 1
except (IOError, OSError):
# logfile may have disappeared
pass
return count
def make_kprs(logdict, kprdict, problem_list):
"""Create kpr files, as necessary, so every log file has one
kpr entries are e.g.
fail/xorg-docs_1:1.6-1.log broken_symlinks_error.conf"""
needs_kpr = set(logdict.keys()).difference(set(kprdict.keys()))
for pkg_spec in needs_kpr:
logpath = logdict[pkg_spec]
try:
lb = open(logpath, 'r')
logbody = lb.read()
lb.close()
where = get_where(logpath)
kf = open(get_kpr_path(logpath), 'a')
for problem in problem_list:
if problem.has_problem(logbody, where):
kf.write("%s/%s.log %s\n" % (where, pkg_spec, problem.name))
kf.close()
except IOError:
print "File error processing %s" % logpath
return len(needs_kpr)
def populate_tpl(tmpl, vals):
for key in vals:
tmpl = re.sub("\$%s" % key, str(vals[key]), tmpl)
return tmpl
def update_tpl(basedir, section, problem, failures, logdict, ftpl, ptpl, pkgsdb):
pkg_text = ""
bugged_section = False
for failure in failures:
pkgspec = failure.pkgspec
bin_pkg = get_pkg(pkgspec)
pkg_obj = pkgsdb.get_package(bin_pkg)
rdep_cnt = pkgsdb.rrdep_count(pkg_obj)
if not pkg_obj is None:
src_pkg = source_pkg(pkgspec, pkgsdb)
else:
src_pkg = bin_pkg
if bugged_section is False and get_where(logdict[pkgspec]) != 'fail':
bugged_section = True
pkg_text += "\n"
pkg_text += populate_tpl(ftpl, {
'LOG': section_path(logdict[pkgspec]),
'PACKAGE': bin_pkg,
'BUG': get_bug_text(logdict[pkgspec]),
'RDEPS': rdep_cnt,
'SDIR':pts_subdir(src_pkg),
'SPKG':src_pkg,
})
if len(pkg_text):
pf = open(os.path.join(basedir, failures[0].problem[:-5] + TPL_EXT), 'w')
tpl_text = populate_tpl(ptpl, {
'HEADER': problem.HEADER,
'SECTION': section,
'HELPTEXT': problem.HELPTEXT,
'COMMAND': problem.get_command(),
'PACKAGE_LIST': pkg_text,
'COUNT': len(failures),
})
pf.write(tpl_text)
pf.close()
def update_html(section, logdict, problem_list, failures, config, pkgsdb):
html_dir = os.path.join(config['output-directory'], section)
if not os.path.exists(html_dir):
os.makedirs(html_dir)
for problem in problem_list:
update_tpl(html_dir, section, problem,
failures.filtered(problem.name),
logdict,
PKG_ERROR_TPL, PROB_TPL, pkgsdb)
# Make a failure list of all failed packages that don't show up as known
failedpkgs = set([x for x in logdict.keys()
if get_where(logdict[x]) != 'pass'])
knownfailpkgs = set([failure.pkgspec for failure in failures.failures])
unknownsasfailures = [make_failure("", "unknown_failures.conf", x)
for x in failedpkgs.difference(knownfailpkgs)]
def keyfunc(x, pkgsdb=pkgsdb, logdict=logdict):
pkg_obj = pkgsdb.get_package(get_pkg(x.pkgspec))
rdeps = pkgsdb.rrdep_count(pkg_obj)
is_failed = get_where(logdict[x.pkgspec]) == "fail"
return (not is_failed, -rdeps, logdict[x.pkgspec])
unknownsasfailures.sort(key=keyfunc)
update_tpl(html_dir, section, problem_list[0], unknownsasfailures,
logdict,
PKG_ERROR_TPL, UNKNOWN_TPL, pkgsdb)
def process_section(section, config, problem_list,
recheck=False, recheck_failed=False, pkgsdb=None):
""" Update .bug and .kpr files for logs in this section """
# raises MissingSection if the section does not exist in piuparts.conf
section_config = WKE_Section_Config(section)
section_config.read(CONFIG_FILE)
sectiondir = os.path.join(config['master-directory'], section)
workdirs = [os.path.join(sectiondir, x) for x in KPR_DIRS]
if not os.access(sectiondir, os.F_OK):
raise MissingSection("", section)
[os.mkdir(x) for x in workdirs if not os.path.exists(x)]
(logdict, kprdict, bugdict) = [get_file_dict(workdirs, x)
for x in [LOG_EXT, KPR_EXT, BUG_EXT]]
del_cnt = clean_cache_files(logdict, kprdict, recheck, recheck_failed)
clean_cache_files(logdict, bugdict, skipnewer=True)
(kprdict, bugdict) = [get_file_dict(workdirs, x) for x in [KPR_EXT, BUG_EXT]]
add_cnt = make_kprs(logdict, kprdict, problem_list)
if not pkgsdb:
distro_config = piupartslib.conf.DistroConfig(
DISTRO_CONFIG_FILE, section_config["mirror"])
sectiondir = os.path.join(config['master-directory'], section)
pkgsdb = piupartslib.packagesdb.PackagesDB(prefix=sectiondir)
pkgsdb.load_packages_urls(
distro_config.get_packages_urls(
section_config.get_distro(),
section_config.get_area(),
section_config.get_arch()))
failures = FailureManager(logdict)
failures.sort_by_bugged_and_rdeps(pkgsdb)
update_html(section, logdict, problem_list, failures, config, pkgsdb)
return (del_cnt, add_cnt, failures)
def detect_well_known_errors(config, problem_list, recheck, recheck_failed):
for section in config['sections'].split():
try:
print time.strftime("%a %b %2d %H:%M:%S %Z %Y", time.localtime())
print "%s:" % section
(del_cnt, add_cnt, failures) = \
process_section(section, config, problem_list,
recheck, recheck_failed)
print "parsed logfiles: %d removed, %d added" % (del_cnt, add_cnt)
for prob in problem_list:
pcount = len(failures.filtered(prob.name))
if pcount:
print "%7d %s" % (pcount, prob.name)
except MissingSection:
pass
print time.strftime("%a %b %2d %H:%M:%S %Z %Y", time.localtime())
def create_problem_list(pdir):
plist = []
for pfile in [x for x in sorted(os.listdir(pdir)) if x.endswith(".conf")]:
prob = Problem(os.path.join(pdir, pfile))
if prob.valid():
plist.append(prob)
else:
print "Keyword error in %s - skipping" % pfile
return plist
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description="Detect/process well known errors to html",
epilog="""
This script processes all log files against defined "known_problem" files,
caching the problems found, by package, into ".kpr" files. The cached data
is summarized into html ".tpl" files in /, which are then
incorporated by piuparts-report into the final web reports.
""")
parser.add_argument('--recheck', dest='recheck', action='store_true',
help="recheck all log files (delete cache)")
parser.add_argument('--recheck-failed', dest='recheck_failed',
action='store_true',
help="recheck failed log files (delete cache)")
args = parser.parse_args()
conf = WKE_Config()
conf.read(CONFIG_FILE)
if conf["proxy"]:
os.environ["http_proxy"] = conf["proxy"]
problem_list = create_problem_list(conf['known-problem-directory'])
detect_well_known_errors(conf, problem_list, args.recheck,
args.recheck_failed)
# vi:set et ts=4 sw=4 :
piuparts-0.56/master-bin/reclassify_bugged.in 0000775 0000000 0000000 00000002441 12144653075 016270 0 ustar #!/bin/sh
set -e
# Copyright © 2012 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
OLDPWD=$(pwd)
for SECTION in $SECTIONS ; do
get_config_value KEEP_BUGGED $SECTION keep-bugged no
if [ "$KEEP_BUGGED" = "no" ] && [ -d $MASTER/$SECTION/fail ] && [ -d $MASTER/$SECTION/bugged ]; then
cd $MASTER/$SECTION
mv bugged/*.log bugged/*.bug fail/ 2>/dev/null
mv affected/*.log affected/*.bug fail/ 2>/dev/null
cd "$OLDPWD"
fi
done
@sharedir@/piuparts/master/report_newly_bugged_packages
piuparts-0.56/master-bin/report_stale_reserved_packages.in 0000775 0000000 0000000 00000002715 12155061511 021041 0 ustar #!/bin/sh
# Copyright 2009,2010 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2012 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
#
# find packages which have been reserved for more than one day and unschedule them
#
DAYS=1
LOGS=`mktemp`
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION/reserved || continue
find $MASTER/$SECTION/reserved/ -mtime +$DAYS -name "*.log" 2>/dev/null >> $LOGS
done
if [ -s $LOGS ] ; then
echo "Stale reserved packages detected, which have been reserved more then $DAYS days ago!"
echo "These packages have been unscheduled."
echo
for package_log in $(cat $LOGS) ; do
rm -fv $package_log
done
fi
rm $LOGS
piuparts-0.56/master-bin/master_cleanup.in 0000775 0000000 0000000 00000002175 12144653075 015615 0 ustar #!/bin/sh
set -e
# Copyright © 2012 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# cleanup $HTDOCS/daily.lock
#
# - this should only be run (automatically) on boot
#
. @sharedir@/piuparts/lib/read_config.sh
get_config_value HTDOCS global output-directory
LOCKFILE=$HTDOCS/daily.lock
if [ -e $LOCKFILE ]; then
if pgrep -f generate_daily_report || pgrep -f piuparts-report ; then
echo "daily processing is running"
else
rm -f -v $LOCKFILE
fi
fi
piuparts-0.56/master-bin/detect_archive_issues.in 0000775 0000000 0000000 00000005257 12144653075 017163 0 ustar #!/bin/sh
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
get_config_value HTDOCS global output-directory
get_config_value URLBASE global urlbase http://piuparts.debian.org
get_config_value DAYS global reschedule-untestable-days 7
#
# detect packages which are/were untestable due to archive issue and mark them as such
#
LOGS=`mktemp`
URLS=`mktemp`
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
mkdir -p $MASTER/$SECTION/fail/ $MASTER/$SECTION/untestable/
find $MASTER/$SECTION/fail -name '*.log' -mtime +1 | xargs -r \
grep -l -E "E: Broken packages|E: Unable to correct problems, you have held broken packages|E: Error, pkgProblemResolver::Resolve generated breaks" 2>/dev/null > $LOGS
if [ -s $LOGS ]; then
for package_log in $(cat $LOGS)
do
mv $package_log $MASTER/$SECTION/untestable/
done
sed "s#$MASTER/$SECTION/fail#$URLBASE/$SECTION/untestable#" $LOGS >> $URLS
fi
done
if [ -s $URLS ]; then
mkdir -p $HTDOCS
date >> $HTDOCS/archive_issues.txt
cat $URLS >> $HTDOCS/archive_issues.txt
echo "Broken packages detected!"
echo "(By grep'ing for"
echo " 'E: Broken packages',"
echo " 'E: Unable to correct problems, you have held broken packages',"
echo " 'E: Error, pkgProblemResolver::Resolve generated breaks'"
echo "in failed logs.)"
echo
echo 'The following packages have been moved to $section/untestable and will be'
echo "tested again in $DAYS days."
echo
echo "Broken packages are usually a temporary problem in the archive and are"
echo "caught by other tools like britney or http://edos.debian.net/edos-debcheck/"
echo
echo "If it is always the same package failing, it's likely to be an issue in the"
echo "package."
echo
grep -f $URLS $HTDOCS/archive_issues.txt | sort | uniq -c | sort -rn
echo
fi
rm $LOGS $URLS
piuparts-0.56/master-bin/prepare_backup.in 0000775 0000000 0000000 00000003315 12155061511 015561 0 ustar #!/bin/sh
set -e
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# backup statistiscs files to a single directory for DSA to backup
# piuparts.d.o's state is not backupped
# (currently it takes one month to run a full piuparts test on a suite
# which produces a gigabyte of logfiles to be saved, for basically
# not much value)
#
. @sharedir@/piuparts/lib/read_config.sh
get_config_value SECTIONS global sections
get_config_value MASTER global master-directory
get_config_value HTDOCS global output-directory
get_config_value BACKUPDIR global backup-directory ''
test -n "$BACKUPDIR" || exit 0
if [ -f $MASTER/bts_stats.txt ]; then
cp $MASTER/bts_stats.txt $BACKUPDIR/
fi
for SECTION in $SECTIONS ; do
mkdir -p $BACKUPDIR/$SECTION
if [ -f $MASTER/$SECTION/submissions.txt ]; then
cp $MASTER/$SECTION/submissions.txt $BACKUPDIR/$SECTION/
fi
if [ -f $HTDOCS/$SECTION/counts.txt ]; then
cp $HTDOCS/$SECTION/counts.txt $BACKUPDIR/$SECTION/
fi
done
piuparts-0.56/master-bin/reschedule_oldest_logs.in 0000775 0000000 0000000 00000011731 12253565512 017331 0 ustar #!/bin/sh
# Copyright 2009-2011 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2012 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
#
# reschedule 200 oldest log files, if they are older than 180 days
# reschedule 25 oldest fail log files, if they are older than 30 days
# delete $reschedule-old-count oldest logfiles, if they are scheduled
# for recycling and older than $expire-old-days
# and $expire-old-days > $reschedule-old-days
#
get_config_value EXPIRE_AGE global expire-old-days 0
get_config_value AGE global reschedule-old-days 180
get_config_value COUNT global reschedule-old-count 200
get_config_value EXPIRE_FAIL_AGE global expire-fail-days 0
get_config_value FAIL_AGE global reschedule-fail-days 30
get_config_value FAIL_COUNT global reschedule-fail-count 25
get_config_value AUTO_RESCHEDULE global auto-reschedule yes
list_logs()
{
__AGE="$1"
__COUNT="$2"
shift 2
find "$@" -name "*.log" -mtime +$__AGE \
| xargs --no-run-if-empty -n99999 -s999999 ls -dt \
| tail -n $__COUNT
}
TOTAL=0
TOTAL_EXPIRED=0
UNSCHEDULE=0
TOTAL_QUEUED=0
LOGS=$(mktemp)
OBSOLETE=$(mktemp)
EXPIRED=$(mktemp)
UNSORTED=$(mktemp)
QUEUED=$(mktemp)
OLDPWD=$(pwd)
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
cd $MASTER/$SECTION
mkdir -p pass fail bugged affected recycle
# Clean up obsolete rescheduling requests
for log in $(find recycle/ -name '*.log' | cut -d"/" -f2) ; do
for dir in pass bugged affected fail ; do
test ! -e "$dir/$log" || continue 2
done
echo "recycle/$log"
done | sort > $OBSOLETE
# Reschedule old logs
>$LOGS
>$EXPIRED
get_config_value RESCHEDULE $SECTION auto-reschedule $AUTO_RESCHEDULE
if [ "$RESCHEDULE" = "yes" ]; then
get_config_value _EXPIRE_AGE $SECTION expire-old-days $EXPIRE_AGE
get_config_value _AGE $SECTION reschedule-old-days $AGE
get_config_value _COUNT $SECTION reschedule-old-count $COUNT
get_config_value _EXPIRE_FAIL_AGE $SECTION expire-fail-days $EXPIRE_FAIL_AGE
get_config_value _FAIL_AGE $SECTION reschedule-fail-days $FAIL_AGE
get_config_value _FAIL_COUNT $SECTION reschedule-fail-count $FAIL_COUNT
# FIXME: we ignore bugged here - ptyhon-bts is really the way to go
>$UNSORTED
if [ "$_EXPIRE_AGE" -gt "$_AGE" ]; then
list_logs $_EXPIRE_AGE $_COUNT pass fail affected >> $UNSORTED
fi
if [ "$_EXPIRE_FAIL_AGE" -gt "$_FAIL_AGE" ]; then
list_logs $_EXPIRE_FAIL_AGE $_FAIL_COUNT fail affected >> $UNSORTED
fi
for log in $(sort -u $UNSORTED) ; do
# the log needs to be scheduled for recycling before it gets expired
test -f "recycle/${log#*/}" && echo "$log"
done > $EXPIRED
>$UNSORTED
list_logs $_AGE $_COUNT pass fail affected >> $UNSORTED
list_logs $_FAIL_AGE $_FAIL_COUNT fail affected >> $UNSORTED
for log in $(sort -u $UNSORTED) ; do
# skip if already scheduled
test -f "recycle/${log#*/}" || echo "$log"
done > $LOGS
fi
find recycle/ -name '*.log' > $QUEUED
if [ -s $LOGS ] || [ -s $OBSOLETE ] || [ -s $EXPIRED ] || [ -s $QUEUED ] ; then
RCOUNT=$(wc -l $LOGS | awk '{ print $1 }')
TOTAL=$(($TOTAL + $RCOUNT))
ECOUNT=$(wc -l $EXPIRED | awk '{ print $1 }')
TOTAL_EXPIRED=$(($TOTAL_EXPIRED + $ECOUNT))
UCOUNT=$(wc -l $OBSOLETE | awk '{ print $1 }')
UNSCHEDULE=$(($UNSCHEDULE + $UCOUNT))
echo "$SECTION: $RCOUNT rescheduled, $ECOUNT expired, $UCOUNT obsolete"
if [ -s $LOGS ]; then
ls -dtl $(cat $LOGS)
ln -f $(cat $LOGS) recycle/
fi
if [ -s $EXPIRED ]; then
ls -dtl $(cat $EXPIRED)
rm -fv $(cat $EXPIRED)
fi
if [ -s $OBSOLETE ]; then
rm -fv $(cat $OBSOLETE)
fi
find recycle/ -name '*.log' > $QUEUED
NUM_QUEUED=$(wc -l < $QUEUED)
TOTAL_QUEUED=$(($TOTAL_QUEUED + $NUM_QUEUED))
echo "queued: $NUM_QUEUED"
echo
echo "#########################################################"
echo
fi
cd $OLDPWD
done
rm $LOGS
rm $OBSOLETE
rm $EXPIRED
rm $UNSORTED
rm $QUEUED
if [ "$TOTAL" -gt "0" ]; then
echo "Rescheduled $TOTAL logs."
fi
if [ "$TOTAL_EXPIRED" -gt "0" ]; then
echo "Deleted $TOTAL_EXPIRED expired logs."
fi
if [ "$UNSCHEDULE" -gt "0" ]; then
echo "Cancelled $UNSCHEDULE outdated rescheduling requests."
fi
if [ "$TOTAL_QUEUED" -gt "0" ]; then
echo "Queued logs: $TOTAL_QUEUED"
fi
piuparts-0.56/master-bin/archive_old_logs.in 0000775 0000000 0000000 00000004255 12145710343 016110 0 ustar #!/bin/sh
# Copyright 2009-2011 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
#
# archive old log files
#
TOTAL=0
OLDPWD=$(pwd)
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
cd $MASTER/$SECTION
mkdir -p pass bugged affected fail
mkdir -p archive/pass archive/bugged archive/affected archive/fail
test -f archive/stamp || touch -d @0 archive/stamp # start at the epoch
touch -d yesterday archive/stamp.new # look back one more day the next time we will be run
OUTPUT=""
# loop through all packages logs
for PACKAGE in $(find pass/ fail/ bugged/ affected/ -name '*.log' -newer archive/stamp | cut -d"_" -f1 | cut -d"/" -f2 | sort -u) ; do
# all logs except the last one (|sed '$d' deletes the last line)
OLDLOGS=$( ls -tr1 --color=none bugged/${PACKAGE}_*.log affected/${PACKAGE}_*.log fail/${PACKAGE}_*.log pass/${PACKAGE}_*.log 2>/dev/null|sed '$d' )
if [ ! -z "$OLDLOGS" ] ; then
if [ -z "$OUTPUT" ] ; then
OUTPUT="yes"
echo $SECTION
fi
for LOG in $OLDLOGS ; do
TOTAL=$(($TOTAL + 1))
mv -v $LOG archive/$(echo $LOG|cut -d"/" -f1)/
done
fi
done
find archive/ -name '*.log' | xargs -r xz -f
if [ -n "$OUTPUT" ] ; then
echo
echo
fi
mv archive/stamp.new archive/stamp
cd "$OLDPWD"
done
if [ "$TOTAL" -gt "0" ]; then
echo "Archived $TOTAL old logfiles."
fi
piuparts-0.56/master-bin/generate_daily_report.in 0000775 0000000 0000000 00000005116 12253565512 017157 0 ustar #!/bin/sh
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
get_config_value HTDOCS global output-directory
get_config_value URLBASE global urlbase http://piuparts.debian.org
get_config_value PYTHONPATH global PYTHONPATH ''
export PYTHONPATH
LOCKFILE=$HTDOCS/daily.lock
if [ -e $LOCKFILE ]; then
echo "daily.lock exists:"
ls -l $LOCKFILE
exit 1
fi
date > $LOCKFILE
#
# various scripts to detect issues and cleanup
#
OUTPUT=$(mktemp)
for SCRIPT in detect_network_issues detect_piuparts_issues archive_old_logs report_newly_bugged_packages ; do
@sharedir@/piuparts/master/$SCRIPT 2>&1 > $OUTPUT
if [ -s $OUTPUT ] ; then
cat $OUTPUT | mail -s $SCRIPT $LOGNAME
fi
done
rm $OUTPUT
#
# piuparts-report starts here
#
DAILYREPORT=$HTDOCS/dailyreport.txt
>$DAILYREPORT
# Failures of the day it should be, the rest is boring.
FAILURESTAMP=$HTDOCS/last-failure-report.stamp
test -f $FAILURESTAMP || touch -d @0 $FAILURESTAMP # start at the epoch
touch $FAILURESTAMP.new # for the next report
echo "New failures:" >> $DAILYREPORT
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
for DIRECTORY in fail bugged affected untestable ; do
mkdir -p $MASTER/$SECTION/$DIRECTORY
done
find $MASTER/$SECTION/fail $MASTER/$SECTION/bugged $MASTER/$SECTION/affected $MASTER/$SECTION/untestable \
-type f -name '*.log' -newer $FAILURESTAMP -exec ls -1 {} + 2>/dev/null
done | sed s#^$MASTER#$URLBASE# >> $DAILYREPORT
echo "" >> $DAILYREPORT
date >> $DAILYREPORT
@sharedir@/piuparts/master/detect_well_known_errors 2>&1 >> $DAILYREPORT
echo "" >> $DAILYREPORT
date >> $DAILYREPORT
nice \
@sharedir@/piuparts/piuparts-report \
>> $DAILYREPORT 2>&1
date >> $DAILYREPORT
cat $DAILYREPORT | mail -s piuparts-report $LOGNAME
mv $FAILURESTAMP.new $FAILURESTAMP
rm -f $LOCKFILE
piuparts-0.56/master-bin/gather_bts_stats.in 0000775 0000000 0000000 00000003357 12155061511 016144 0 ustar #!/bin/bash
set -e
# Copyright 2013 Holger Levsen (holger@layer-acht.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
BTS_STATS="$MASTER/bts_stats.txt"
# exit if master-directory doesn't exist or if devscripts package is not installed
test -n "$MASTER" || $(which bts) || exit 0
# "bts select" needs libsoap-lite-perl too
dpkg -l libsoap-lite-perl >/dev/null 2>&1 || exit 0
# only run once a day
TODAY=$(date +%Y%m%d)
if $(grep -q ^$TODAY $BTS_STATS 2>/dev/null) ; then
exit 0
fi
# query bts
ALL=$(bts select usertag:piuparts users:debian-qa@lists.debian.org archive:both 2>/dev/null|wc -l)
OPEN=$(bts select usertag:piuparts users:debian-qa@lists.debian.org status:open 2>/dev/null|wc -l)
# test if both values are integers
if ! ( [[ $ALL =~ ^-?[0-9]+$ ]] && [[ $OPEN =~ ^-?[0-9]+$ ]] ) ; then
echo "Non-integer value detected, exiting. ALL: $ALL OPEN: $OPEN"
fi
# init file if needed
if [ ! -f $BTS_STATS ] ; then
echo "date, all, open" > $BTS_STATS
fi
# finally, write stats
echo "$TODAY, $ALL, $OPEN" >> $BTS_STATS
piuparts-0.56/master-bin/report_untestable_packages.in 0000775 0000000 0000000 00000003042 12144653075 020204 0 ustar #!/bin/sh
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2012 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
get_config_value DAYS global reschedule-untestable-days 7
#
# find packages which have been in untestable for more than $DAYS days and reschedule them for testing
#
LOGS=`mktemp`
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION/untestable || continue
find $MASTER/$SECTION/untestable/ -mtime +$DAYS -name "*.log" 2>/dev/null >> $LOGS
done
if [ -s $LOGS ] ; then
echo "Untestable packages detected, which have been tested more than $DAYS days ago!"
echo "These packages have been rescheduled for piuparts testing."
echo
for package_log in $(cat $LOGS) ; do
rm -fv $package_log
done
fi
rm $LOGS
piuparts-0.56/master-bin/detect_piuparts_issues.in 0000775 0000000 0000000 00000007066 12253565512 017410 0 ustar #!/bin/sh
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
#
# detect piuparts problems
#
FILE=`mktemp`
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
for subdir in fail bugged affected ; do
test -d $MASTER/$SECTION/$subdir || continue
rgrep -l 'PIUPARTS OUTPUT INCOMPLETE' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -E "tar( \(child\))?: .*.tar.gz: Cannot open: No such file or directory" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "tar: .*: No space left on device" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "tar: Error is not recoverable: exiting now" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "gzip: stdin: invalid compressed data--crc error" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "ERROR:.*:Temporary directory is not a directory" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "No space left on device" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "E: You don't have enough free space in /var/cache/apt/archives/" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "^RuntimeError: maximum recursion depth exceeded while calling a Python object" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "^Traceback" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "^OSError:" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "^IndentationError:" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "^SyntaxError:" $MASTER/$SECTION/$subdir >> $FILE
# Python errors from maintainer scripts etc. would be indented by 2 spaces, so don't match here.
rgrep -l -e 'update-binfmts: warning: unable to close /proc/sys/fs/binfmt_misc/register: Invalid argument' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e 'userdel: user .* is currently logged in' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e 'invoke-rc.d: initscript mysql, action "start" failed.' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e 'invoke-rc.d: initscript postgresql, action "start" failed.' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e 'No database found online on port 5432' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e 'unable to connect to postgresql server' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e 'Firebird .* server already running.' $MASTER/$SECTION/$subdir >> $FILE
done
done
if [ -s $FILE ] ; then
echo "piuparts problem detected!"
echo "(By grep'ing for 'tar: .*.tar.gz: Cannot open: No such file or directory'"
echo "and for some python errors and tracebacks in failed logs.)"
echo
echo "Please review the following logfiles/packages."
echo "If it is always the same package failing, it's likely to be an issue in the"
echo "package."
echo
sort -u $FILE | xargs rm -v
echo
echo "----------------------------------------------------------------------"
echo
fi
rm $FILE
piuparts-0.56/master-bin/detect_network_issues.in 0000775 0000000 0000000 00000006422 12253565512 017225 0 ustar #!/bin/sh
# Copyright 2009 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
#
# detect network/mirror problems
#
FILE=`mktemp`
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
for subdir in fail bugged affected ; do
test -d $MASTER/$SECTION/$subdir || continue
rgrep -l "Cannot initiate the connection to" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l "Hash Sum mismatch" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "Failed to fetch.*Could not resolve" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "Failed to fetch.*Something wicked happened resolving" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "Failed to fetch.*Unable to connect to" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l "E: Method http has died unexpectedly" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l "Some index files failed to download, they have been ignored, or old ones used instead." $MASTER/$SECTION/$subdir >> $FILE
rgrep -l "Some index files failed to download. They have been ignored, or old ones used instead." $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -e "W: GPG error: .* Release: The following signatures were invalid: BADSIG" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing" $MASTER/$SECTION/$subdir >> $FILE
rgrep -l 'E: Method file has died unexpectedly!' $MASTER/$SECTION/$subdir >> $FILE
rgrep -l "E: Sub-process rred received a segmentation fault." $MASTER/$SECTION/$subdir >> $FILE
rgrep -l -E "ERROR: Command failed \(status=-7\):.*'apt-get', 'update'" $MASTER/$SECTION/$subdir >> $FILE
# modified changelogs are usually caused by mirror pushes during the piuparts test
rgrep -l -E ' /usr/share/doc/(.*)/changelog.*owned by: \1' $MASTER/$SECTION/$subdir >> $FILE
done
done
if [ -s $FILE ] ; then
echo "Network problems on detected!"
echo "(By grep'ing for"
echo " 'Cannot initiate the connection to',"
echo " 'Failed to fetch',"
echo " 'Some index files failed to download',"
echo " 'Hash Sum mismatch',"
echo " 'The following signatures were invalid'"
echo "in failed logs.)"
echo "Test failures due to modified /usr/share/doc/*/changelog.*"
echo "are caused by mirror updates during the test."
echo
echo "The following logfiles have been deleted:"
echo
echo "----------------------------------------------------------------------"
echo
for LOG in $(sort -u $FILE) ; do
rm -v $LOG
done
echo
fi
rm $FILE
piuparts-0.56/master-bin/report_newly_bugged_packages.in 0000775 0000000 0000000 00000002625 12253565512 020516 0 ustar #!/bin/sh
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value SECTIONS global sections
get_config_value PYTHONPATH global PYTHONPATH ''
get_config_value DEBEMAIL global bts-from ''
export PYTHONPATH
OLDPWD=$(pwd)
OUTPUT=$(mktemp)
for SECTION in $SECTIONS ; do
test -d $MASTER/$SECTION || continue
cd $MASTER/$SECTION
get_config_value DISTROS $SECTION upgrade-test-distros ''
distro=${DISTROS##* }
distro=${distro%-proposed}
timeout 60m @sharedir@/piuparts/piuparts-analyze $distro 2>&1 > $OUTPUT
if [ -s $OUTPUT ]; then
echo $SECTION
cat $OUTPUT
echo
fi
cd "$OLDPWD"
done
rm $OUTPUT
piuparts-0.56/piuparts-analyze.py 0000664 0000000 0000000 00000022705 12155061511 014070 0 ustar #!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2005 Lars Wirzenius (liw@iki.fi)
# Copyright 2011 Mika Pflüger (debian@mikapflueger.de)
# Copyright © 2012-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
"""Analyze failed piuparts logs and move them around when the errors are known.
This program looks at piuparts log files in ./fail, and queries the bts to find
out if bugs have been filed already. If so, it moves them to ./bugged.
It tries to detect if new versions of bugged packages are uploaded without solving
the bug and will then update the bts with the new found versions, and copy the
headers of the log in ./fail to the one in ./bugged and vice versa. It will then
move the failed log to ./bugged as well.
"""
import os
import sys
import re
import shutil
import subprocess
import debianbts
import apt_pkg
apt_pkg.init_system()
error_pattern = re.compile(r"(?<=\n).*error.*\n?", flags=re.IGNORECASE)
chroot_pattern = re.compile(r"tmp/tmp.*?'")
distro = "?"
def find_logs(directory):
return [os.path.join(directory, x)
for x in os.listdir(directory) if x.endswith(".log")]
def find_bugged_logs(failed_log):
package = package_name(failed_log)
pat = "/" + package + "_"
return [x for x in find_logs("bugged") + find_logs("affected") if pat in x]
def package_name(log):
return os.path.basename(log).split("_", 1)[0]
def package_version(log):
return os.path.basename(log).split("_", 1)[1].rstrip('.log')
def package_source_version(log):
version = package_version(log)
possible_binnmu_part = version.rsplit('+', 1)[-1]
if possible_binnmu_part.startswith('b') and possible_binnmu_part[1:].isdigit():
# the package version contains a binnmu-part which is not part of the source version
# and therefore not accepted/tracked by the bts. Remove it.
version = version.rsplit('+', 1)[0]
return version
def extract_errors(log):
"""This pretty stupid implementation is basically just 'grep -i error', and then
removing the timestamps and the name of the chroot and the package version itself."""
f = open(log)
data = f.read()
f.close()
whole = ''
pversion = package_version(log)
for match in error_pattern.finditer(data):
text = match.group()
# Get rid of timestamps
if text[:1].isdigit():
text = text.split(" ", 1)[1]
# Get rid of chroot names
if 'tmp/tmp' in text:
text = re.sub(chroot_pattern, "chroot'", text)
# Get rid of the package version
text = text.replace(pversion, '')
whole += text
return whole
def extract_headers(log):
f = open(log)
data = f.read()
f.close()
headers = []
headers = data.partition("\nExecuting:")[0]
if headers and not headers.endswith("\n"):
headers += "\n"
return headers
def prepend_to_file(filename, data):
f = file(filename, "r")
old_data = f.read()
f.close()
f = file(filename + ".tmp", "w")
f.write(data)
f.write(old_data)
f.close()
shutil.copymode(filename, filename + ".tmp")
os.rename(filename, filename + "~")
os.rename(filename + ".tmp", filename)
os.remove(filename + "~")
def get_bug_versions(bug):
"""Gets a list of only the version numbers for which the bug is found.
Newest versions are returned first."""
# debianbts returns it in the format package/1.2.3 or 1.2.3 which will become 1.2.3
versions = []
for found_version in debianbts.get_status((bug,))[0].found_versions:
v = found_version.rsplit('/', 1)[-1]
if v == "None":
# only allow the distro-qualified "$distro/None" version
if found_version == distro + "/" + v:
versions.append(v)
else:
versions.append(v)
return list(reversed(sorted(versions, cmp=apt_pkg.version_compare))) or ['~']
def write_bug_file(failed_log, bugs):
if bugs:
f = file(os.path.splitext(failed_log)[0] + '.bug', "w")
for bug in bugs:
f.write('#%s \n' % (bug, bug))
f.close()
def move_to_bugged(failed_log, bugged="bugged", bug=None):
print("Moving %s to %s (#%s)" % (failed_log, bugged, bug))
os.rename(failed_log, os.path.join(bugged, os.path.basename(failed_log)))
if bug is not None:
write_bug_file(os.path.join(bugged, os.path.basename(failed_log)), [bug])
def mark_bugged_version(failed_log, bugged_log):
"""Copies the headers from the old log to the new log and vice versa and
moves the new log to bugged. Removes the old log in bugged."""
bugged_headers = extract_headers(bugged_log)
failed_headers = extract_headers(failed_log)
prepend_to_file(failed_log, bugged_headers)
prepend_to_file(bugged_log, failed_headers)
move_to_bugged(failed_log)
def bts_update_found(bugnr, newversion):
if "DEBEMAIL" in os.environ and os.environ["DEBEMAIL"]:
#subprocess.check_call(('bts', 'found', bugnr, newversion))
print(' '.join(('bts', 'found', str(bugnr), newversion)))
def mark_logs_with_reported_bugs():
for failed_log in find_logs("fail"):
pname = package_name(failed_log)
pversion = package_source_version(failed_log)
failed_errors = extract_errors(failed_log)
moved = False
abugs = piuparts_bugs_affecting(pname)
bugs = piuparts_bugs_in(pname)
for bug in abugs + bugs:
if moved:
break
if bug in abugs:
bugged = "affected"
else:
bugged = "bugged"
found_versions = get_bug_versions(bug)
if pversion in found_versions:
move_to_bugged(failed_log, bugged, bug)
moved = True
break
for bug_version in found_versions:
#print('DEBUG: %s/%s #%d %s' % (pname, pversion, bug, bug_version))
if apt_pkg.version_compare(pversion, bug_version) > 0: # pversion > bug_version
bugged_logs = find_bugged_logs(failed_log)
if not bugged_logs and not moved:
print('%s/%s: Maybe the bug was filed earlier: http://bugs.debian.org/%d against %s/%s'
% (pname, pversion, bug, pname, bug_version))
break
for bugged_log in bugged_logs:
old_pversion = package_source_version(bugged_log)
bugged_errors = extract_errors(bugged_log)
if (apt_pkg.version_compare(old_pversion, bug_version) == 0 # old_pversion == bug_version
and
failed_errors == bugged_errors):
# a bug was filed for an old version of the package,
# and the errors were the same back then - assume it is the same bug.
if not moved:
mark_bugged_version(failed_log, bugged_log)
moved = True
bts_update_found(bug, pversion)
break
if not moved:
write_bug_file(failed_log, abugs + bugs)
def report_packages_with_many_logs():
failed_logs = find_logs("fail")
packages = {}
for failed_log in failed_logs:
package = package_name(failed_log)
packages[package] = packages.get(package, []) + [failed_log]
for package, failed_logs in packages.iteritems():
printed = False
if len(failed_logs) > 1:
print "Many failures:"
for failed_log in failed_logs:
print " ", failed_log
printed = True
bugged_logs = find_bugged_logs(failed_logs[0])
if bugged_logs:
print "Already bugged?"
for failed_log in failed_logs + bugged_logs:
print " ", failed_log
printed = True
if printed:
print
piuparts_usertags_cache = None
def all_piuparts_bugs():
global piuparts_usertags_cache
if piuparts_usertags_cache is None:
piuparts_usertags_cache = debianbts.get_usertag("debian-qa@lists.debian.org", 'piuparts')['piuparts']
return piuparts_usertags_cache
def piuparts_bugs_in(package):
bugs = debianbts.get_bugs('package', package, 'bugs', all_piuparts_bugs(), 'archive', 'both')
bugs.sort(reverse=True)
return bugs
def piuparts_bugs_affecting(package):
bugs = debianbts.get_bugs('affects', package, 'bugs', all_piuparts_bugs(), 'archive', 'both')
bugs.sort(reverse=True)
return bugs
def main():
if len(sys.argv) > 1:
global distro
distro = sys.argv[1]
mark_logs_with_reported_bugs()
report_packages_with_many_logs()
if __name__ == "__main__":
main()
# vi:set et ts=4 sw=4 :
piuparts-0.56/update-piuparts-slave-setup 0000775 0000000 0000000 00000004610 12145710343 015527 0 ustar #!/bin/sh
set -e
#
# update piuparts slave setup from git (eg. used on piu-slave-bm-a.debian.org)
#
# Copyright 2009-2013 Holger Levsen (holger@layer-acht.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
if [ "`id -n -u`" != "piupartss" ] ; then
echo please run this script as piupartss user
exit 1
fi
PIUPARTS_PREFIX=/srv/piuparts.debian.org
PIUPARTS_HTDOCS=$PIUPARTS_PREFIX/htdocs
PIUPARTS_TMPDIR=$PIUPARTS_PREFIX/tmp
#
# create $PIUPARTS_PREFIX
#
if [ ! -d $PIUPARTS_PREFIX ] ; then
sudo mkdir -p $PIUPARTS_PREFIX
sudo chown piupartss:piuparts $PIUPARTS_PREFIX
sudo chmod 0775 $PIUPARTS_PREFIX
fi
#
# update source
#
if [ ! -d $PIUPARTS_PREFIX/src ] ; then
mkdir -p $PIUPARTS_PREFIX/src
chmod 0755 $PIUPARTS_PREFIX/src
cd $PIUPARTS_PREFIX/src
git clone git://git.debian.org/git/piuparts/piuparts.git
cd piuparts
git checkout develop
fi
cd $PIUPARTS_PREFIX/src/piuparts
pwd
# git checkout branch if $1 is given
if [ ! -z "$1" ] ; then
git checkout $1
fi
# git fetch+pull if $2 is given
if [ ! -z "$2" ] ; then
git fetch $2
git pull $2 $1
fi
#
# install everything from GIT into PIUPARTS_PREFIX
#
make clean
make prefix=$PIUPARTS_PREFIX \
build build-doc
make prefix=$PIUPARTS_PREFIX \
docdir=$PIUPARTS_HTDOCS/doc \
htdocsdir=$PIUPARTS_HTDOCS \
install install-doc install-conf-4-running-from-git
make clean
#
# update $PIUPARTS_PREFIX
#
cd $PIUPARTS_PREFIX
pwd
mkdir -p slave slave/basetgz
# to support multiple hosts with this setup
cd etc/piuparts
HOSTNAME=$(hostname)
for f in piuparts.conf
do
ln -sf $f.$HOSTNAME $f
done
#
# create working dir
#
mkdir -p $PIUPARTS_TMPDIR
#
# update slave home
#
cd
pwd
ln -sf $PIUPARTS_PREFIX/share/piuparts/slave bin
crontab $PIUPARTS_PREFIX/etc/piuparts/crontab-slave
echo "Update finished."
piuparts-0.56/piuparts_slave_run.8.txt 0000664 0000000 0000000 00000003137 12155061511 015040 0 ustar piuparts_slave_run(8)
=====================
:doctype: manpage
NAME
----
piuparts_slave_run - start up piuparts-slave instance(s) in screen
SYNOPSIS
--------
*piuparts_slave_run* ['SLAVE-NUMBERS']
DESCRIPTION
-----------
*piuparts_slave_run* runs piuparts-slave instance(s) in *screen* which run *piuparts* tests which test whether Debian packages handle installation, upgrading, and removal correctly.
By default, *piuparts_slave_run* will start the number of slaves specified by the 'slave-count' parameter in the piuparts.conf file. If 'slave-count' is not defined, a single slave will be started. If a single argument is supplied to *piuparts_slave_run*, consisting of a space-separated list of numbers between 1 and 'slave-count', only those slaves will be started.
OPTIONS
-------
There are no options to this command currently. Note however that *this interface is under development* and may change.
ENVIRONMENT
-----------
Running piuparts in master-slave mode requires configuration in _/etc/piuparts_.
BUGS
----
*piuparts_slave_run* should be able to run slaves in different ways:
1. all, in a new screen (only this is implemented at the moment)
2. a specific one, in a specified screen slot in a (running) screen
3. append one to a (running) screen using a "random" slot
4. start slave in the current directory in the current shell in foreground.
NOTES
-----
Make sure to also read */usr/share/doc/piuparts-master/README_server.txt*.
SEE ALSO
--------
*piuparts*(1), *piuparts_slave_join*(8), *screen*(1)
AUTHOR
------
Holger Levsen (holger@layer-acht.org)
DATE
----
2013-05-27
// vim: set filetype=asciidoc:
piuparts-0.56/README_server.txt 0000664 0000000 0000000 00000053576 12253565512 013315 0 ustar piuparts README_server
----------------------
Author: Lars Wirzenius, Holger Levsen and Andreas Beckmann
Email:
== piuparts in master/slave mode
As part of the quality assurance effort of Debian, piuparts is
run on the Debian package archive. This requires a lot of
processing power, and so the work can be distributed over several
hosts.
There is one central machine, the master, and any number of slave
machines. Each slave machine connects to the master, via ssh, and
runs the piuparts-master program to report results of packages it
has tested already, and to get more work.
To set this up for yourself, the following steps should suffice:
. Pick a machine to run the master. It cannot be a chroot, but
basically any real (or properly virtualized) Debian system is good
enough.
. Install the package piuparts-master on it.
. Create an account for the master (the package creates piupartsm).
. Configure '/etc/piuparts/piuparts.conf' appropriately.
. Pick one or more slaves to run the slave. You can use the machine
running the master also as a slave. Etch is fine, it can even be
in a chroot.
. Install the package piuparts-slave on it.
. Configure '/etc/piuparts/piuparts.conf' appropriately - if master
and slave share the machine, they also share the config file.
. Create an account for the slave. This must be different from the
master account. (The package creates piupartss.)
. Create an ssh keypair for the slave. No passphrase.
. Add the slave's public key to the master's '.ssh/authorized_keys'
The key should be restricted to only allow running
'piuparts-master' by prefixing it with
'command="/usr/share/piuparts/piuparts-master",no-pty,no-port-forwarding'
. Configure sudo (via '/etc/sudoers.d/piuparts') on the slave machine(s)
to allow the slave account to run '/usr/sbin/piuparts' as root without
password (otherwise you'll be typing in a password all the time).
. Run '/usr/share/piuparts/piuparts-slave' on the slave accounts.
Packages that are installed want to use '/dev/tty', so you can't
do this from cron. Also, you'll want to keep an eye on what is
happening, to catch runaway processes and stuff.
. The logs go into the master account, into subdirectories.
=== Setup from piuparts-master and piuparts-slaves packages
The piuparts-server package installs a piuparts server along the lines of
http://piuparts.debian.org/.
Before running the server, edit /etc/piuparts.conf appropriately (install
piuparts-slave (which ships that file), too, or use the template
/usr/share/doc/piuparts-master/piuparts.conf.sample), to define
'sections' to be tested (e.g. 'sid') and define references to the Debian
mirror. Note that the server can place a significant load on the
repository. Consider setting up a local mirror, or a caching proxy for http
and apt-get, to reduce the load. Running multiple slaves on a fast host can
easily saturate a 100 MBit link.
Edit '/etc/sudoers.d/piuparts' to grant permissions to the piupartss user.
Start the server using /usr/sbin/piuparts_slave_run, which will launch a
'screen' session. The slave will launch a master process via ssh, as needed,
to retrieve work and return results. Use /usr/sbin/piuparts_slave_join to
join the screen session.
Logs are stored under /var/lib/piuparts. They are stored there because they
are basically the result of piuparts running.
There are maintenance cron jobs defined in /etc/cron.d/piuparts-*.cron. In
particular, piuparts-report will create a web summary, defaulting to
http://localhost/piuparts, served by Apache. Uncomment the lines in the cron
file to enable the jobs.
=== Setup from GIT
http://piuparts.debian.org has been set up directly from GIT, this is
described in '/usr/share/doc/piuparts-master/README_pejacevic.txt'.
== Distributed testing
WARNING: Please note that running piuparts this way is somewhat risky, to
say the least. There are security implications that you want to
consider. It's best to do it on machines that you don't mind
wiping clean at a moment's notice, and preferably so that they
don't have direct network access.
=== Distributed piuparts testing protocol
The slave machine and the piuparts-master program communicate
using a simplistic line based protocol. SSH takes care of
authentication, so there is nothing in the protocol for that. The
protocol is transaction based: the slave gives a command, the
master program responds. Commands and responses can be simple (a
single line) or long (a status line plus additional data lines).
Simple commands and responses are of the following format:
'keyword arg1 arg2 arg3 ... argN'
The keyword is a command or status code ("ok"), and it and the
arguments are separated by spaces. An argument may not contain a
space.
A long command or response is deduced from the context: certain
commands always include additional data, and certain commands
always get a long response, if successful (error responses are
always simple). The first line of a long command or response is
the same as for a simple one, the additional lines are prefixed
with a space, and followed by a line containing only a period.
A sample session (">>" indicates what the slave sends, "<<" what
the master responds with):
----
<< hello
>> section sid
<< ok
>> pass liwc 1.2.3-4
>> The piuparts
>> log file comes
>> here
>> .
<< ok
>> reserve
<< ok vorbisgain 2.3-4
----
Here the slave first reports a successful test of package liwc,
version 1.2.3-4, and sends the piuparts log file for it. Then it
reserves a new package to test and the master gives it
vorbisgain, version 2.3-4.
The communication always starts with the master saying "hello".
The slave shall not speak until the master has spoken.
Commands and responses in this protocol:
----
Command: section
Success: ok
Failure: error
Failure: busy
----
Slave asks master to select the given section.
This must be the very first command sent by the slave, but may
be repeated later on to switch between sections.
It will return "error" if the section is unknown and "busy" if
it is currently processed by another master instance. If the
section command fails, no other commands than "section" will be
allowed until one succeeds.
----
Command: recycle
Success: ok
Failure: error
----
Slave asks master to enable logfile recycling mode. In this mode
logfiles that have been marked for rechecking will be deleted
and reissued in subsequent "reserve" commands. The "recycle"
command must be issued before the first "reserve" (or "status")
command. It will return "error" if no more logfiles are marked
for rechecking or the command is issued too late.
----
Command: idle
Success: ok
----
Slave asks master whether it remembers having no packages
available at a previous "reserve" command. Returns 0 (not known
to be idle or timeout expired) or the number of seconds until
the master wants to recompute the package state. This command
should be given after "recycle" and logfile submission, but
before "reserve" or "status" commands. If the slave closes the
connection without issuing a "reserve" or "status" command, the
expensive Packages file parsing and status computation will be
skipped.
----
Command: reserve
Success: ok
Failure: error
----
Slave asks master to reserve a package (a particular version of
it) for the slave to test. The slave may reserve any number of
packages to test. If the transaction fails, there are no more
packages to test, and the slave should disconnect, wait some time
and try again.
----
Command: unreserve
Success: ok
----
Slave informs master it cannot test the desired version of a
package and the package should be rescheduled by the master.
----
Command: pass
log file contents
.
Success: ok
----
Slave reports that it has tested a particular version of a
package and that the package passed all tests. Master records
this and stores the log file somewhere suitable.
----
Command: fail
log file contents
.
Success: ok
----
Same as "pass", but package failed one or more tests.
----
Command: untestable
log file contents
.
Success: ok
----
Slave informs master it cannot test the desired version of a
package (perhaps it went away from the mirror?).
----
Command: status
Success: ok = =...
----
Slave asks master to report the number of packages in all
different states. The "status" command should only be issued
after all logs have been transmitted ("pass", "fail", and
"untestable" commands).
In all cases, if the master cannot respond with "ok" (e.g.,
because of a disk error storing a log file), it aborts and the
connection fails. The slave may only assume the command has
succeeded if the master responds with "ok".
The master may likewise abort, without an error message, if the
slave sends garbage, or sends too much data.
=== piuparts.conf configuration file
piuparts-master, piuparts-slave and piuparts-report share the
configuration file '/etc/piuparts/piuparts.conf'. The syntax is
defined by the Python ConfigParser class, and is, briefly, like
this:
----
[master]
foo = bar
----
==== global configuration
These settings have to be placed in the [global] section and are
used for all further sections.
* "sections" defaults to sid and defines which sections should be
processed in master-slave mode. Each section defined here has to
have a section with the section specific settings explained below.
The first section defined should always be sid, because the data
from first section a package is in is used for the source package
html report.
* "master-host" is the host where the master exists. The slave will
give this host to ssh. This option is mandatory.
* "master-user" is the username of the master. The slave will log in
using this username. This option is mandatory.
* "master-directory" is the directory where the master keeps its
files. Can be relative to the master's home directory.
* "slave-directory" is the directory where the slave keeps its
files. Can be relative to the slave's home directory.
* "slave-count" is the number of concurrent slaves to start.
Default: "1".
* "output-directory" is the directory where piuparts-report places
the logfiles, generated html files, charts, ... that can be
served by a webserver.
* "backup-directory" is the directory where the prepare_backup
script will place copies of the history data needed to generate the
plots. This directory should be included in system backups while
the logfiles and html pages in 'master-directory' and
'output-directory' (several GB of data) are regeneratable with some
effort and can be excluded from backups. By default this is
undefined meaning that no backups of the history data will be made.
* "doc-root" is the location where the webserver will serve the
piuparts report from. Default: "/".
* "slave-load-max" specifies the system load limit when
piuparts-slave will enter sleep mode. Operation will be resumed
after load drops below 'slave-load-max - 1.0'. Floating point
value. Defaults to 0 (= disabled).
* "proxy" sets the http_proxy that will be used for fetching
Packages files etc. (by master/slave/report) and .debs etc. (by
piuparts). This will override a http_proxy setting in the
environment. By default (no value being set) the http_proxy
variable from the environment will be used (and no proxy will be
used if this is not set). It is highly recommended to use a proxy
running on localhost (e.g. installing squid and using a setting of
"http://localhost:3128") due to the high bandwidth consumption of
piuparts and repeated downloading of the same files.
==== section specific configuration
The section specific settings will be reloaded each time a section
is being run. All these keys can be specified in the [global]
section, too, and will serve as defaults for all other sections
(overriding the builtin defaults).
* "master-command" is the command to run on master-host to start
the master. When the master has been installed from the Debian
package, the command is '/usr/share/piuparts/piuparts-master'.
This does not need to be set here if it is already set in
'~piupartsm/.ssh/authorized_keys' to limit ssh access to that
single command.
* "idle-sleep" is the length of time the slave should wait before
querying the master again if the master didn't have any new
packages to test. In seconds, so a value of 300 would mean five
minutes, and that seems to be a good value when there are fairly
few slaves per master. The default is 300 seconds.
* "max-tgz-age" is used to specify the maximum age (in seconds)
after which basesystem tarballs will be recreated. If recreation
fails, the old tarball will be used again. The default is 2592000
seconds, which is 30 days. A value of 0 disables recreation.
* "min-tgz-retry-delay" is used to specify the minimum time (in
seconds) between attempts to recreate a tarball which was created
more than "max-tgz-age" seconds ago. The default is 21600 seconds,
which is 6h.
* "log-file" is the name of a file to where the master should write
its log messages. In the default configuration file it is
"$SECTION/master.log". To disable logging, set it to "/dev/null".
The global "log-file" setting (defaulting to master-error.log) is
used for logging stderr output from piuparts-master. This logfile
will be placed in the 'master-directory' and has the PID appended.
* "piuparts-command" is the command the slave uses to start
piuparts. It should include 'sudo' if necessary so that piuparts
runs with sufficient priviledges to do its testing (and that
means root priviledges). This command should be given in the
[global] section and include all flags that are common for all
sections.
* "piuparts-flags" are appended to "piuparts-command" and should
contain the section-specific flags.
* "tmpdir" is the scratch area where piuparts will create the
chroots. Note: the filesystem where this is located must not be
mounted with the nodev or nosuid options. This is a mandatory
setting with no default. The scripts that are monitoring this
directory for leftover mountpoints and chroots only evaluate the
[global] setting.
* "description" is a synopsis of the test used in the report. A
default description will be generated if this is not set or will
be prepended (appended) if the description starts (ends) with
'+'.
* "mirror" tells the slave which mirror it is to use. The slave
gives this to piuparts when it runs it. The URLs for Packages and
Sources files will be generated from this setting, too. Default
(for fetching Packages/Sources): "http://cdn.debian.net/debian".
* "distro" is the distribution the slave should tell piuparts to
use for basic install/purge testing. It is also possible to use a
"partial" distribution as defined in distros.conf. No default.
If 'upgrade-test-distros' is set, this selects the distribution
that will be used for getting the packages to be tested. Defaults
to the last entry in 'upgrade-test-distros', but other useful
settings are the first entry (to test upgrades of "disappearing"
packages) or the restricted set in a partial distribution (e.g.
stable to backports to testing).
* "area" is the archive area used to get the list of packages to
be tested. The Packages file for this area will be loaded. The
default is "main" and the possible values depend on the vendor,
for Debian these are main, contrib, non-free.
* "components" sets the archive areas that will be available when
testing the packages selected via the "area" setting. These will
be enabled in the generated sources.list. Defaults to "", which
means all components will be available. A useful setting is
"main" together with area = main to avoid using packages outside
main. Testing packages from a 'partial' area like contrib or
non-free usually requires additional or all components to be
available.
* "arch" is the architecture to use.
Default: dpkg --print-architecture.
* "chroot-tgz" is the name of the file the slave should use for
the tarball containing the base chroot. The default name is
generated automatically from the "distro" or "upgrade-test-distros"
setting. If the tarball doesn't exist, the slave creates it.
* "basetgz-directory" is the directory where "chroot-tgz" (or the
automatically selected default name) is located. The default is
'.'.
* "upgrade-test-distros" is the space delimited list of
distributions the slave should use for testing upgrades
between distributions (i.e., Debian versions). Using "partial"
distributions as defined in distros.conf is possible. Currently,
"squeeze wheezy sid" is a good choice.
Setting this switches from doing install/purge tests to
dist-upgrade tests. Not set by default.
* "max-reserved" is the maximum number of packages the slave will
reserve at once. It should be large enough that the host that
runs master is not unduly stressed by frequent ssh logins and
running master (both of which take quite a bit of CPU cycles),
yet at the same time it should not be so large that one slave
grabs so many packages all other slaves just sit idle. The number
obviously depends on the speed of the slave. A good value seems
to be enough to let the slave test packages for about an hour
before reporting results and reserving more. For a contemporary
AMD64 machine with a reasonably fast disk subsystem the value 50
seems to work fine. To disable a section set this to 0.
* "keep-sources-list" controls whether the slave runs piuparts
with the '--keep-sources-list' option. This option does not
apply to upgrade tests. The value should be "yes" or "no", with
the default being "no". Use this option for dists that you need
a custom sources.list for, such as "stable-proposed-updates".
* "precedence" controls the order the sections are being processed
by the slave. Sections with a larger precedence value will be run
only if all sections with a smaller precedence value are idle,
i.e. master does not have any packages that this slave could
test. Sections with the same precedence value will be processed
round-robin until they are all idle (or a more important section
has packages to be tested). The default is 1.
* "depends-sections" lists additional sections that will be
searched for dependencies that are not available in the current
section if that describes a partial distro.
* "known-problem-directory" is the path to the directory containing
definitions of known problems.
Default: "${prefix}/share/piuparts/known_problems"
* "debug" tells the slave whether to log debug level messages. The
value should be "yes" or "no", with the default being "no".
piuparts itself currently always produces debug output and there
is no way to disable that.
* "PYTHONPATH" (global) sets the search path to the piupartslib
python modules if they are not installed in their default location
in /usr.
* "reschedule-untestable-days" (global) sets the rescheduling
delay for untestable packages (e.g. due to unsatisfied
dependencies). This is handled by the 'report_untestable_packages'
script and the default is "7" days.
* "reschedule-old-days" (global, section) and the following five
settings define the rescheduling scheme that it performed by the
'reschedule_oldest_logs' script. Passed/failed logs that are
older than reschedule-(old|fail)-days will be marked for
rechecking (limited to reschedule-(old|fail)-count). Only packages
that are actually testable will be reissued by piuparts-master (and
the "old" log will be deleted at that time). Logs that are marked
for recycling but have not been rechecked due to missing/failing
dependecies will be deleted anyway if they are older than
expire-(old|fail)-days.
* "reschedule-old-count" (global, section) is the maximum number of
passed logs that will be marked for recycling. Set to 0 to disable
rescheduling passed logs.
* "expire-old-days" (global, section) can be set to a value larger
than 'reschedule-old-days' to delete logs older than the setting
that are marked for recycling but haven't been rechecked due to
failing or missing dependecies. Disabled by default ("0").
* "reschedule-fail-days" (global, section) sets the minimum age of
failing logs (fail/*.log or affected/*.log) before they will be
rechecked.
* "reschedule-fail-count" (global, section) is the maximum number
of failed logs that will be marked for recycling. Set to 0 to
disable rescheduling failed logs.
* "expire-fail-days" (global, section) can be set to a value larger
than 'reschedule-fail-days' to delete logs older than the setting
that are marked for recycling but haven't been rechecked due to
failing or missing dependecies. Disabled by default ("0").
* "auto-reschedule" (section) can be set to "no" to disable
rescheduling of passed and failed packages. To disable only
rescheduling one of passed or failed logs, set the corresponding
-count variable to zero.
Some of the configuration items are not required, but it is best
to set them all to be sure what the configuration actually is.
==== piuparts.debian.org specific configuration
In addition to some of the above settings the following
configuration settings are used by the scripts in '~piuparts?/bin/'
used to run piuparts.debian.org. They are all optional, default
values are set in the scripts.
* "urlbase" (global) is the base url of the webserver serving this
piuparts instance. Used to provide links to logfiles in email
reports.
=== Running piuparts in master-slave mode, piuparts-report and the setup on piuparts.debian.org
If you want to run piuparts-report (which is only+very useful if
you run piuparts in master-slave mode), you need to 'apt-get
install python-rpy r-recommended r-base-dev'. For more
information see
link:http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git;hb=master;a=blob;f=README_server.txt[http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git;hb=master;a=blob;f=README_server.txt] and
link:http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git;hb=master;a=blob;f=README_pejacevic.txt[http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git;hb=master;a=blob;f=README_pejacevic.txt].
// vim: set filetype=asciidoc:
piuparts-0.56/piuparts-report.py 0000664 0000000 0000000 00000166360 12253565512 013757 0 ustar #!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2005 Lars Wirzenius (liw@iki.fi)
# Copyright 2009-2013 Holger Levsen (holger@layer-acht.org)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
"""Create HTML reports of piuparts log files
Lars Wirzenius
"""
import os
import sys
import time
import logging
import ConfigParser
import shutil
import re
import string
import yaml
import hashlib
# if python-rpy2 ain't installed, we don't draw fancy graphs
try:
from rpy2 import robjects
from rpy2.robjects.packages import importr
except:
pass
import piupartslib
from piupartslib.conf import MissingSection
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
PIUPARTS_VERSION = "__PIUPARTS_VERSION__"
HTML_HEADER = """
$page_title
"""
LOG_LIST_BODY_TEMPLATE = """
$title in $section
$preface
The list has $count packages, with $versioncount total versions.
$logrows
"""
STATE_BODY_TEMPLATE = """
Packages in state "$state" in $section $aside
"""
SECTION_INDEX_BODY_TEMPLATE = """
$section statistics
$description
Binary packages per state
$tablerows
URL to Packages file
$packagesurl
"""
MAINTAINER_BODY_TEMPLATE = """
$maintainer
$distrolinks
$rows
"""
SOURCE_PACKAGE_BODY_TEMPLATE = """
"""
ANALYSIS_BODY_TEMPLATE = """
"""
title_by_dir = {
"pass": "PASSED piuparts logs",
"fail": "Failed UNREPORTED piuparts logs",
"bugged": "Failed REPORTED piuparts logs",
"affected": "Failed AFFECTED piuparts logs",
"reserved": "RESERVED packages",
"untestable": "UNTESTABLE packages",
}
desc_by_dir = {
"pass": "Log files for packages that have PASSED testing.",
"fail": "Log files for packages that have FAILED testing. " +
"Bugs have not yet been reported.",
"bugged": "Log files for packages that have FAILED testing. " +
"Bugs have been reported, but not yet fixed.",
"affected": "Log files for packages that have dependencies FAILED testing. " +
"Bugs have been reported, but not yet fixed.",
"reserved": "Packages that are RESERVED for testing on a node in a " +
"distributed piuparts network.",
"untestable": "Log files for packages that have are UNTESTABLE with " +
"piuparts at the current time.",
}
state_by_dir = {
"pass": "successfully-tested",
"fail": "failed-testing",
"bugged": "failed-testing",
"affected": "failed-testing",
"reserved": "waiting-to-be-tested",
"untestable": "cannot-be-tested",
}
# better use XX_name.tpl and get the linktarget from the template
# (its a substring of the of the that template
# maintaining this list is errorprone and tiresome
linktarget_by_template = [
("initdscript_lsb_header_issue.tpl", "but logfile contains update-rc.d issues"),
("command_not_found_issue.tpl", "but logfile contains 'command not found'"),
("debsums_mismatch_issue.tpl", "but logfile contains modified conffiles or other shipped files"),
("alternatives_after_purge_issue.tpl", "but logfile contains forgotten alternatives"),
("owned_files_after_purge_issue.tpl", "but logfile contains owned files existing after purge"),
("unowned_files_after_purge_issue.tpl", "but logfile contains unowned files after purge"),
("maintainer_script_issue.tpl", "but logfile contains maintainer script failures"),
("db_setup_issue.tpl", "but logfile contains failure to setup a database"),
("installs_over_symlink_issue.tpl", "but package installs something over existing symlinks"),
("broken_symlinks_issue.tpl", "but logfile contains 'broken symlinks'"),
("unknown_inadequate_issue.tpl", "but logfile contains unknown inadequate issues"),
("boring_obsolete_conffile_file_inadequate_issue.tpl", "...and logfile contains tag from adequate 'obsolete-conffile-file'"),
("boring_broken_symlink_file_inadequate_issue.tpl", "...and logfile contains tag from adequate 'broken-symlink-file'"),
("bin_or_sbin_binary_requires_usr_lib_library_inadequate_issue.tpl", "but adequate found a binary in /bin or /sbin that requires a /usr/lib library"),
("library_not_found_inadequate_issue.tpl", "but adequate couldn't find a required library"),
("ldd_inadequate_issue.tpl", "but adequate encountered unexpected ldd warnings"),
("incompatible_licenses_inadequate_issue.tpl", "but adequate found a license incompatibility"),
("missing_copyright_file_inadequate_issue.tpl", "but adequate couldn't find a copyright file"),
("py_file_not_bytecompiled_inadequate_issue.tpl", "but adequate found a .py file that is not byte-compiled"),
("pyshared_file_not_bytecompiled_inadequate_issue.tpl", "but adequate found a .py file in /usr/share/pyshared that is not byte-compiled"),
("undefined_symbol_inadequate_issue.tpl", "but adequate found an undefined symbol"),
("symbol-size-mismatch_inadequate_issue.tpl", "but adequate found that a symbol has changed size since the package was built"),
("missing-symbol-version-information_inadequate_issue.tpl", "but adequate found that a library is missing symbol version information"),
("unknown_inadequate_issue.conf", "but an unknown adequate issue was found"),
("inadequate_exit_issue.tpl", "but adequate exited inadequately"),
("packages_have_been_kept_back_issue.tpl", "but logfile contains 'packages have been kept back'"),
("needs_rebuild_issue.tpl", "but logfile recommends to rebuild some packages"),
("obsolete_conffiles_issue.tpl", "but logfile reports obsolete conffiles"),
("missing_md5sums_issue.tpl", "but logfile reports missing md5sums"),
("used_exception_issue.tpl", "but package used a piuparts exception"),
("dependency_error.tpl", "due to unsatisfied dependencies"),
("packages_have_been_kept_back_error.tpl", "...and logfile also contains 'packages have been kept back'"),
("command_not_found_error.tpl", "due to a 'command not found' error"),
("files_in_usr_local_error.tpl", "due to files in /usr/local"),
("overwrite_other_packages_files_error.tpl", "due to overwriting other packages files"),
("debsums_mismatch_error.tpl", "due to modifying conffiles or other shipped files"),
("alternatives_after_purge_error.tpl", "due to forgotten alternatives after purge"),
("owned_files_by_many_packages_error.tpl", "due to owned files by many packages"),
("owned_files_after_purge_error.tpl", "due to owned files existing after purge"),
("unowned_files_after_purge_error.tpl", "due to unowned files after purge"),
("modified_files_after_purge_error.tpl", "due to files having been modified after purge"),
("disappeared_files_after_purge_error.tpl", "due to files having disappeared after purge"),
("diversion_error.tpl", "due to diversions being modified after purge"),
("processes_running_error.tpl", "due to leaving processes running behind"),
("resource_violation_error.tpl", "due to resource violation"),
("conffile_prompt_error.tpl", "due to prompting due to modified conffiles"),
("db_setup_error.tpl", "due to failing to setup a database"),
("insserv_error.tpl", "due to a problem with insserv"),
("problems_and_no_force_error.tpl", "due to not enough force being used"),
("immediate_configuration_error.tpl", "due to apt could not perform immediate configuration"),
("pre_depends_error.tpl", "due to a problem with pre-depends"),
("pre_installation_script_error.tpl", "due to pre-installation maintainer script failed"),
("post_installation_script_error.tpl", "due to post-installation maintainer script failed"),
("pre_removal_script_error.tpl", "due to pre-removal maintainer script failed"),
("post_removal_script_error.tpl", "due to post-removal maintainer script failed"),
("unknown_purge_error.tpl", "due to purge failed due to an unknown reason"),
("cron_error_after_removal_error.tpl", "due to errors from cronjob after removal"),
("logrotate_error_after_removal_error.tpl", "due to errors from logrotate after removal"),
("installs_over_symlink_error.tpl", "...and package installs something over existing symlinks"),
("broken_symlinks_error.tpl", "...and logfile also contains 'broken symlinks'"),
("obsolete_conffiles_error.tpl", "...and logfile reports obsolete conffiles"),
("missing_md5sums_error.tpl", "...and logfile reports missing md5sums"),
("unknown_failures.tpl", "due to unclassified failures"),
]
class Config(piupartslib.conf.Config):
def __init__(self, section="report", defaults_section=None):
self.section = section
piupartslib.conf.Config.__init__(self, section,
{
"sections": "report",
"output-directory": "html",
"master-directory": ".",
"depends-sections": None,
"description": "",
"proxy": None,
"mirror": None,
"distro": None,
"area": None,
"arch": None,
"upgrade-test-distros": None,
"max-reserved": 1,
"doc-root": "/",
"known-problem-directory": "@sharedir@/piuparts/known_problems",
},
defaults_section=defaults_section)
def setup_logging(log_level, log_file_name):
logger = logging.getLogger()
logger.setLevel(log_level)
formatter = logging.Formatter(fmt="%(asctime)s %(message)s",
datefmt="%H:%M:%S")
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(formatter)
logger.addHandler(handler)
if log_file_name:
handler = logging.FileHandler(log_file_name)
logger.addHandler(handler)
def html_protect(vstr):
vstr = "&".join(vstr.split("&"))
vstr = "<".join(vstr.split("<"))
vstr = ">".join(vstr.split(">"))
vstr = """.join(vstr.split('"'))
vstr = "'".join(vstr.split("'"))
return vstr
def emphasize_reason(reason):
bad_states = [
#"successfully-tested",
"failed-testing",
"cannot-be-tested",
#"essential-required",
#"waiting-to-be-tested",
#"waiting-for-dependency-to-be-tested",
"dependency-failed-testing",
"dependency-cannot-be-tested",
"dependency-does-not-exist",
"circular-dependency", # obsolete
"unknown",
"unknown-preferred-alternative", # obsolete
"no-dependency-from-alternatives-exists", # obsolete
"does-not-exist",
]
if reason in bad_states:
reason = ""+reason+" "
return reason
def source_subdir(source):
if source[:3] == "lib":
return source[:4]
else:
return source[:1]
def maintainer_subdir(maintainer):
return maintainer.lower()[:1]
def find_files_with_suffix(vdir, suffix):
pairs = [] # (mtime, name)
for name in os.listdir(vdir):
if name.endswith(suffix):
try:
if os.path.isfile(os.path.join(vdir, name)):
mtime = os.path.getmtime(os.path.join(vdir, name))
pairs.append((mtime, name))
except OSError:
pass
# sort by mtime
return [x[1] for x in sorted(pairs)]
def update_file(source, target):
if os.path.exists(target):
try:
aa = os.stat(source)
bb = os.stat(target)
except OSError:
pass
else:
if aa.st_size == bb.st_size and aa.st_mtime <= bb.st_mtime:
return
try:
os.remove(target)
except:
pass
try:
os.link(source, target)
except OSError:
try:
shutil.copyfile(source, target)
except IOError as (errno, strerror):
logging.error("failed to copy %s to %s: I/O error(%d): %s" \
% (source, target, errno, strerror))
def copy_logs(logs_by_dir, output_dir):
for vdir in logs_by_dir:
fulldir = os.path.join(output_dir, vdir)
if not os.path.exists(fulldir):
os.makedirs(fulldir)
for basename in logs_by_dir[vdir]:
source = os.path.join(vdir, basename)
target = os.path.join(fulldir, basename)
update_file(source, target)
def remove_old_logs(logs_by_dir, output_dir):
for vdir in logs_by_dir:
fulldir = os.path.join(output_dir, vdir)
# convert logs_by_dir array to a dict to avoid linear search
logs_dict = {}
for log in logs_by_dir[vdir]:
logs_dict[log] = 1
if os.path.exists(fulldir):
for basename in os.listdir(fulldir):
if not basename in logs_dict:
os.remove(os.path.join(fulldir, basename))
def write_file(filename, contents):
f = file(filename, "w")
f.write(contents)
f.close()
def append_file(filename, contents):
f = file(filename, "a")
f.write(contents)
f.close()
def read_file(filename):
f = file(filename, "r")
l = f.readlines()
f.close()
return l
def write_template_html(filename, body, mapping={}):
header = HTML_HEADER
footer = HTML_FOOTER
htmlpage = string.Template(header + body + footer)
mapping = mapping.copy()
mapping.update({
"content_md5": "",
"piuparts_version": "",
"time": "",
})
content_md5 = hashlib.md5(htmlpage.safe_substitute(mapping)).hexdigest()
mapping.update({
"content_md5": content_md5,
"piuparts_version": PIUPARTS_VERSION,
"time": time.strftime("%Y-%m-%d %H:%M %Z"),
})
write_file(filename, htmlpage.safe_substitute(mapping))
def create_section_navigation(section_names, current_section, doc_root):
tablerows = ""
for section in section_names:
tablerows += ("%s \n") % \
(doc_root, html_protect(section), html_protect(section))
tablerows += "by maintainer / uploader \n" \
% (doc_root, current_section)
tablerows += "by source package \n" \
% (doc_root, current_section)
return tablerows;
def get_email_address(maintainer):
email = "INVALID maintainer address: %s" % (maintainer)
try:
m = re.match(r"(.+)(<)(.+@.+)(>)", maintainer)
email = m.group(3)
except:
pass
return email
def package2id(package_name):
# "+" is not a valid identifier char for id=... attributes
return package_name.replace("+", "_")
# return order preserving list of the first occurrence of an element
def unique(stuff):
# can't use set() because 'stuff' is a list of lists and list() is not hashable
vlist = []
previtem = stuff
for item in stuff:
if item != previtem:
vlist.append(item)
previtem = item
return vlist
class Section:
def __init__(self, section, master_directory, doc_root, packagedb_cache={}):
self._config = Config(section=section, defaults_section="global")
self._config.read(CONFIG_FILE)
self._distro_config = piupartslib.conf.DistroConfig(
DISTRO_CONFIG_FILE, self._config["mirror"])
logging.debug("-------------------------------------------")
logging.debug("Running section " + self._config.section)
self._section_directory = os.path.abspath(os.path.join(master_directory, \
self._config.section))
if not os.path.exists(self._section_directory):
logging.debug("Warning: %s did not exist, now created. Did you ever let the slave work?"
% self._section_directory)
os.makedirs(self._section_directory)
self._doc_root = doc_root
logging.debug("Loading and parsing Packages file")
self._packagedb_cache = packagedb_cache
self._package_databases = {}
self._load_package_database(section, master_directory)
self._binary_db = self._package_databases[section]
self._source_db = piupartslib.packagesdb.PackagesDB()
self._source_db.load_packages_urls(
self._distro_config.get_sources_urls(
self._config.get_distro(),
self._config.get_area()))
self._log_name_cache = {}
def _load_package_database(self, section, master_directory):
if section in self._package_databases:
return
elif section in self._packagedb_cache:
self._package_databases[section] = self._packagedb_cache[section]
return
config = Config(section=section, defaults_section="global")
config.read(CONFIG_FILE)
if not config["depends-sections"]:
# this is a base database eligible for caching
# only cache the most recent base database
self._packagedb_cache.clear()
sectiondir = os.path.join(master_directory, section)
db = piupartslib.packagesdb.PackagesDB(prefix=sectiondir)
self._package_databases[section] = db
if config["depends-sections"]:
deps = config["depends-sections"].split()
for dep in deps:
self._load_package_database(dep, master_directory)
db.set_dependency_databases([self._package_databases[dep] for dep in deps])
else:
# only cache the big base databases that don't have additional dependencies
self._packagedb_cache[section] = db
db.load_packages_urls(
self._distro_config.get_packages_urls(
config.get_distro(),
config.get_area(),
config.get_arch()))
if config.get_distro() != config.get_final_distro():
# take version numbers (or None) from final distro
db2 = piupartslib.packagesdb.PackagesFile()
db2.load_packages_urls(
self._distro_config.get_packages_urls(
config.get_final_distro(),
config.get_area(),
config.get_arch()))
for package in db.get_all_packages():
if package["Package"] in db2:
package["Version"] = db2[package["Package"]]["Version"]
else:
package["Version"] = "None"
def _write_template_html(self, filename, body, mapping={}):
mapping = mapping.copy()
mapping.update({
"section_navigation": self._section_navigation,
"doc_root": self._doc_root,
"section": html_protect(self._config.section),
})
write_template_html(filename, body, mapping)
def write_log_list_page(self, filename, title, preface, logs):
packages = {}
for pathname, package, version in logs:
packages[package] = packages.get(package, []) + [(pathname, version)]
names = packages.keys()
names.sort()
lines = []
version_count = 0
for package in names:
versions = []
for pathname, version in packages[package]:
version_count += 1
versions.append("%s " %
(html_protect(pathname),
html_protect(version)))
line = "%s %s " % \
(html_protect(package),
", ".join(versions))
lines.append(line)
if "FAIL" in preface:
title_style="alerttitlecell"
else:
title_style="titlecell"
self._write_template_html(
filename,
LOG_LIST_BODY_TEMPLATE,
{
"page_title": html_protect(title+" in "+self._config.section),
"title": html_protect(title),
"title_style": title_style,
"preface": preface,
"count": len(packages),
"versioncount": version_count,
"logrows": "".join(lines),
})
def print_by_dir(self, output_directory, logs_by_dir):
for vdir in logs_by_dir:
vlist = []
for basename in logs_by_dir[vdir]:
assert basename.endswith(".log")
assert "_" in basename
package, version = basename[:-len(".log")].split("_")
vlist.append((os.path.join(vdir, basename), package, version))
self.write_log_list_page(os.path.join(output_directory, vdir + ".html"),
title_by_dir[vdir],
desc_by_dir[vdir], vlist)
def find_links_to_logs(self, package_name, dirs, logs_by_dir):
links = []
for vdir in dirs:
# avoid linear search against log file names by caching in a dict
#
# this cache was added to avoid a very expensive linear search
# against the arrays in logs_by_dir. Note that the use of this cache
# assumes that the contents of logs_by_dir is invarient across calls
# to find_links_to_logs()
#
if vdir not in self._log_name_cache:
self._log_name_cache[vdir] = {}
for basename in logs_by_dir[vdir]:
if basename.endswith(".log"):
package, version = basename[:-len(".log")].split("_")
self._log_name_cache[vdir][package] = version
if vdir == "fail":
style = " class=\"needs-bugging\""
else:
style = ""
if package_name in self._log_name_cache[vdir]:
basename = package_name \
+ "_" \
+ self._log_name_cache[vdir][package_name] \
+ ".log"
links.append("%s " % (
self._doc_root,
os.path.join(self._config.section, vdir, basename),
style,
html_protect(self._log_name_cache[vdir][package_name]),
))
return links
def link_to_maintainer_summary(self, maintainer):
email = get_email_address(maintainer)
return "%s " \
% (self._doc_root, self._config.section, maintainer_subdir(email),
email, html_protect(maintainer))
def link_to_uploaders(self, uploaders):
link = ""
for uploader in uploaders.split(","):
link += self.link_to_maintainer_summary(uploader.strip()) + ", "
return link[:-2]
def link_to_source_summary(self, package_name):
source_name = self._binary_db.get_control_header(package_name, "Source")
link = "%s " % (
self._doc_root,
self._config.section,
source_subdir(source_name)+"/"+source_name+".html",
html_protect(package_name))
return link
def link_to_state_page(self, section, package_name, link_target):
if self._binary_db.has_package(package_name):
state = self._binary_db.get_package_state(package_name)
link = "%s " % (
self._doc_root,
section,
"state-"+state+".html"+"#"+package_name,
link_target)
else:
if link_target == package_name:
link = html_protect(package_name)
else:
link = "unknown-package"
return link
def links_to_logs(self, package_name, state, logs_by_dir):
link = "N/A"
dirs = ""
if state == "successfully-tested":
dirs = ["pass"]
elif state == "failed-testing":
dirs = ["fail", "bugged", "affected"]
elif state == "cannot-be-tested":
dirs = ["untestable"]
if dirs != "":
links = self.find_links_to_logs (package_name, dirs, logs_by_dir)
link = ", ".join(links)
if "/bugged/" in link or "/affected/" in link:
link += " - bug filed \n"
return link
def write_counts_summary(self):
logging.debug("Writing counts.txt")
header = "date"
current_day = "%s" % time.strftime("%Y%m%d")
counts = current_day
total = 0
for state in self._binary_db.get_states():
count = len(self._binary_db.get_pkg_names_in_state(state))
header += ", %s" % state
counts += ", %s" % count
logging.debug("%s: %s" % (state, count))
total += count
logging.debug("total: %s" % total)
logging.debug("source: %s" % len(self._source_db.get_all_packages()))
header += "\n"
counts += "\n"
countsfile = os.path.join(self._output_directory, "counts.txt")
if not os.path.isfile(countsfile):
logging.debug("writing new file: %s" % countsfile)
write_file(countsfile, header)
last_line = ""
else:
last_line = read_file(countsfile)[-1]
if not current_day in last_line:
append_file(countsfile, counts)
logging.debug("appending line: %s" % counts.strip())
return total
def create_maintainer_summaries(self, maintainers, source_data):
logging.debug("Writing %d maintainer summaries in %s" % (len(maintainers), self._output_directory))
maintainer_dir = os.path.join(self._output_directory, "maintainer")
if not os.path.exists(maintainer_dir):
os.mkdir(maintainer_dir)
states = ["fail", "unknown", "pass"]
for maintainer in sorted(maintainers.keys()):
sources = maintainers[maintainer]
maintainer_subdir_path = os.path.join(maintainer_dir, maintainer_subdir(maintainer))
if not os.path.exists(maintainer_subdir_path):
os.mkdir(maintainer_subdir_path)
rows = ""
package_rows = {}
packages = {}
for state in states:
packages[state] = []
package_rows[state] = ""
for source in sorted(sources):
(state, sourcerows, binaryrows) = source_data[source]
packages[state].append(source)
package_rows[state] += sourcerows + binaryrows
for state in states:
if len(packages[state]) > 0:
links = ""
for package in packages[state]:
links += "%s " % (package, package)
else:
links = " "
rows += "" \
+ "%s: " % state \
+ "%s " % len(packages[state]) \
+ "%s " % links \
+ " \n"
distrolinks = "" \
+ "other distributions: " \
+ ""
for section in self._section_names:
if section != self._config.section:
distrolinks += "" \
+ html_protect(section) \
+ " "
distrolinks += " "
self._write_template_html(
os.path.join(maintainer_subdir_path, maintainer + ".html"),
MAINTAINER_BODY_TEMPLATE,
{
"page_title": html_protect("Status of " \
+ maintainer \
+ " packages in " \
+ self._config.section),
"maintainer": html_protect(maintainer+" in "+self._config.section),
"distrolinks": distrolinks,
"rows": rows + "".join([package_rows[state] for state in states]),
})
def create_source_summary (self, source, logs_by_dir):
source_version = self._source_db.get_control_header(source, "Version")
binaries = self._source_db.get_control_header(source, "Binary")
maintainer = self._source_db.get_control_header(source, "Maintainer")
uploaders = self._source_db.get_control_header(source, "Uploaders")
success = True
failed = False
binaryrows = ""
for binary in sorted([x.strip() for x in binaries.split(",") if x.strip()]):
if not self._binary_db.has_package(binary):
# udebs or binary packages for other architectures
# The latter is a FIXME which needs parsing the Packages files from other archs too
binaryrows += "" \
+ "Binary: " \
+ "%s " \
% binary \
+ "unknown package " \
+ " \n"
continue
state = self._binary_db.get_package_state(binary)
if not "waiting" in state and "dependency" in state:
state_style="lightalertlabelcell"
elif state == "failed-testing":
state_style="lightlabelcell"
else:
state_style="labelcell"
binary_version = self._binary_db.get_control_header(binary, "Version")
binaryrows += "" \
+ "Binary: " \
+ "%s " \
% binary\
+ "piuparts-result: " \
% state_style \
+ "%s %s " \
% (self.link_to_state_page(self._config.section, binary, state),
self.links_to_logs(binary, state, logs_by_dir)) \
+ "Version: " \
+ "%s " \
% html_protect(binary_version) \
+ " \n"
if state not in ("successfully-tested", "essential-required"):
success = False
if state in ("failed-testing", "dependency-does-not-exist", "cannot-be-tested"):
failed = True
if binaryrows != "":
source_state="unknown"
if success: source_state=" " % self._doc_root
if failed: source_state=" " % self._doc_root
sourcerows = "" \
+ "%s in %s " \
% (package2id(source), source, self._config.section) \
+ " \n"
sourcerows += "" \
+ "Source: " \
+ "" \
+ "%s " \
% (source, html_protect(source)) \
+ " " \
+ "piuparts summary: " \
+ "%s " \
% source_state \
+ "Version: " \
+ "%s " \
% html_protect(source_version) \
+ " \n"
sourcerows += "" \
+ "Maintainer: " \
+ "%s " \
% self.link_to_maintainer_summary(maintainer) \
+ " \n"
if uploaders:
sourcerows += "" \
+ "Uploaders: " \
+ "%s " \
% self.link_to_uploaders(uploaders) \
+ " \n"
source_summary_page_path = os.path.join(self._output_directory,
"source",
source_subdir(source))
if not os.path.exists(source_summary_page_path):
os.makedirs(source_summary_page_path)
self._write_template_html(
os.path.join(source_summary_page_path, (source + ".html")),
SOURCE_PACKAGE_BODY_TEMPLATE,
{
"page_title": html_protect("Status of source package "+source+" in "+self._config.section),
"rows": sourcerows+binaryrows,
})
# return parsable values
if success: source_state = "pass"
if failed: source_state = "fail"
else:
source_state = "udeb"
sourcerows = ""
return sourcerows, binaryrows, source_state, maintainer, uploaders
def create_package_summaries(self, logs_by_dir):
logging.debug("Writing source summaries in %s" % self._config.section)
maintainers = {}
source_binary_rows = {}
sources = ""
sources_data = []
for source in sorted(self._source_db.get_all_package_names()):
(sourcerows, binaryrows, source_state, maintainer, uploaders) = \
self.create_source_summary(source, logs_by_dir)
if source_state != "udeb":
source_version = self._source_db.get_control_header(source, "Version")
source_data = {};
source_data["source"] = source
source_data["state"] = source_state
source_data["version"] = source_version
sources_data.append(source_data)
sources += "%s: %s\n" % (source, source_state)
source_binary_rows[source] = (source_state, sourcerows, binaryrows)
for maint in [maintainer] + uploaders.split(","):
if maint.strip():
email = get_email_address(maint.strip())
if not "INVALID" in email:
if not email in maintainers:
maintainers[email] = []
maintainers[email].append(source)
write_file(os.path.join(self._output_directory, "sources.txt"), sources)
write_file(os.path.join(self._output_directory, "sources.yaml"),
yaml.dump(sources_data, default_flow_style=False))
self.create_maintainer_summaries(maintainers, source_binary_rows)
def make_stats_graph(self):
countsfile = os.path.join(self._output_directory, "counts.txt")
pngfile = os.path.join(self._output_directory, "states.png")
grdevices = importr('grDevices')
grdevices.png(file=pngfile, width=1600, height=900, pointsize=10, res=100, antialias="none")
r = robjects.r
r('t <- (read.table("'+countsfile+'",sep=",",header=1,row.names=1))')
r('cname <- c("date",rep(colnames(t)))')
# here we define how many days we wants stats for (163=half a year)
#r('v <- t[(nrow(t)-163):nrow(t),0:12]')
# make graph since day 1
r('v <- t[0:nrow(t),0:12]')
# thanks to http://tango.freedesktop.org/Generic_Icon_Theme_Guidelines for those nice colors
r('palette(c("#4e9a06", "#ef2929", "#d3d7cf", "#5c3566", "#c4a000", \
"#fce94f", "#a40000", "#888a85", "#2e3436", "#729fcf", \
"#3465a4", "#204a87", "#555753"))')
r('barplot(t(v),col = 1:13, \
main="Binary packages per state in '+self._config.section+'", \
xlab="", ylab="Number of binary packages", space=0, border=NA)')
r('legend(x="bottom",legend=colnames(t), ncol=2,fill=1:13,xjust=0.5,yjust=0,bty="n")')
grdevices.dev_off()
stats_html = " " \
+ "" \
+ "" \
% "states.png" \
+ " " \
% ("states.png", self._config.section) \
+ " \n"
return stats_html
def create_and_link_to_analysises(self, state):
link="\n"
for template, linktarget in linktarget_by_template:
# successful logs only have issues and failed logs only have errors
if (state == "failed-testing" and template[-9:] != "issue.tpl") \
or (state == "successfully-tested" and template[-9:] == "issue.tpl"):
substats = ""
tpl = os.path.join(self._output_directory, template)
try:
f = file(tpl, "r")
rows = file.read(f)
f.close()
os.unlink(tpl)
self._write_template_html(
os.path.join(self._output_directory, template[:-len(".tpl")]+".html"),
ANALYSIS_BODY_TEMPLATE,
{
"page_title": html_protect("Packages in state "+state+" "+linktarget),
"rows": rows,
})
if state == "failed-testing":
count_bugged = string.count(rows, '"bugged/')
count_affected = string.count(rows, '"affected/')
count_failed = string.count(rows, '"fail/')
sep = ": "
if count_bugged > 0:
substats += sep + "%s bugged" % count_bugged
sep = ", "
if count_affected > 0:
substats += sep + "%s affected" % count_affected
sep = ", "
if count_failed > 0:
substats += sep + "%s failed " % count_failed
else:
count_passed = string.count(rows, '"pass/')
if count_passed > 0:
substats += ": %s passed" % count_passed
link += "%s %s \n" % \
(
template[:-len(".tpl")]+".html",
linktarget,
substats,
)
except:
logging.debug("analysis template %s does not exist." % template)
link += " "
if link == "":
link = ""
return link
def write_section_index_page(self, dirs, total_packages):
tablerows = ""
for state in self._binary_db.get_active_states():
dir_link = ""
analysis = ""
for vdir in dirs:
if vdir in ("pass", "fail", "bugged", "affected", "untestable") and state_by_dir[vdir] == state:
dir_link += "%s logs " % (vdir, html_protect(vdir))
if state in ("successfully-tested", "failed-testing"):
analysis = self.create_and_link_to_analysises(state)
tablerows += ("%s %s " +
"%d %s \n") % \
(html_protect(state), html_protect(state), analysis, len(self._binary_db.get_pkg_names_in_state(state)),
dir_link)
try:
tablerows += self.make_stats_graph();
except:
logging.debug("Error generating the graph images, probably python-rpy2 is not installed, disabling graphs.")
tablerows += " Total %d \n" % total_packages
vendor = "Debian"
if len(self._config.get_distros()) > 1:
description = "%s %s: package installation in %s" % (
vendor,
self._config.get_area(),
self._config.get_start_distro())
for distro in self._config.get_distros()[1:]:
description += ", dist-upgrade to %s" % distro
description += ", removal, and purge test."
else:
description = "%s %s / %s: package installation, removal, and purge test." % (
vendor,
self._config.get_distro(),
self._config.get_area())
if self._config["description"].startswith("+"):
description = description + " " + self._config["description"][1:]
elif self._config["description"].endswith("+"):
description = self._config["description"][:-1] + " " + description
elif self._config["description"]:
description = self._config["description"]
self._write_template_html(
os.path.join(self._output_directory, "index.html"),
SECTION_INDEX_BODY_TEMPLATE,
{
"page_title": html_protect(self._config.section+" statistics"),
"description": html_protect(description),
"tablerows": tablerows,
"packagesurl": " ".join([html_protect(url) for url in self._binary_db.get_urls()]),
})
def _show_providers(self, dep):
providers = self._binary_db.get_providers(dep)
vlist = ""
if providers:
vlist += "\n\n"
for provider in providers:
vlist += "provider %s is %s \n" % \
(self.link_to_state_page(self._config.section, provider, provider),
emphasize_reason(html_protect(self._binary_db.get_package_state(provider))))
vlist += " \n"
return vlist
def write_state_pages(self):
for state in self._binary_db.get_active_states():
logging.debug("Writing page for %s" % state)
vlist = ""
if state in self._binary_db.get_error_states():
with_counts = True
aside = " (reverse deps, blocked pkgs)"
sort_key = lambda x: (-self._binary_db.block_count(x), x["Package"])
else:
with_counts = False
aside = ""
sort_key = lambda x: x["Package"]
names = self._binary_db.get_pkg_names_in_state(state)
packages = [self._binary_db.get_package(name) for name in names]
packages.sort(key=sort_key)
for package in packages:
vlist += "%s" % (
package2id(package["Package"]),
self.link_to_source_summary(package["Package"]))
if with_counts:
vlist += " (%d, %d)" % (self._binary_db.rrdep_count(package), \
self._binary_db.block_count(package))
vlist += " (%s)" % html_protect(package["Maintainer"])
all_deps = unique(package.all_dependencies())
if all_deps:
vlist += "\n\n"
for alternatives in all_deps:
dep = alternatives[0]
vlist += "dependency %s is %s" % \
(self.link_to_state_page(self._config.section, dep, dep),
emphasize_reason(html_protect(self._binary_db.get_package_state(dep, resolve_virtual=False))))
vlist += self._show_providers(dep)
if len(alternatives) > 1:
vlist += "\n\n"
for dep in alternatives[1:]:
vlist += "alternative dependency %s is %s" % \
(self.link_to_state_page(self._config.section, dep, dep),
emphasize_reason(html_protect(self._binary_db.get_package_state(dep, resolve_virtual=False))))
vlist += self._show_providers(dep)
vlist += " \n"
vlist += " \n"
vlist += " \n"
vlist += " \n"
vlist += " \n"
self._write_template_html(
os.path.join(self._output_directory, "state-%s.html" % state),
STATE_BODY_TEMPLATE,
{
"page_title": html_protect("Packages in state "+state+" in "+self._config.section),
"state": html_protect(state),
"list": vlist,
"aside": aside,
})
def archive_logfile(self, vdir, log):
archivedir = os.path.join("archive", vdir)
if not os.path.exists(archivedir):
os.makedirs(archivedir)
os.rename(os.path.join(vdir, log), os.path.join("archive", vdir, log))
def cleanup_removed_packages(self, logs_by_dir):
for vdir in logs_by_dir.keys():
for log in sorted(logs_by_dir[vdir]):
if log.endswith(".log"):
package, version = log[:-len(".log")].split("_")
if not self._binary_db.has_package(package):
logging.debug("Package %s was removed, archiving %s/%s" % (package, vdir, log))
self.archive_logfile(vdir, log)
logs_by_dir[vdir].remove(log)
def generate_html(self):
logging.debug("Finding log files")
dirs = ["pass", "fail", "bugged", "affected", "reserved", "untestable"]
logs_by_dir = {}
for vdir in dirs:
logs_by_dir[vdir] = find_files_with_suffix(vdir, ".log")
logging.debug("Archiving logs of obsolete packages")
self.cleanup_removed_packages(logs_by_dir)
logging.debug("Copying log files")
copy_logs(logs_by_dir, self._output_directory)
logging.debug("Removing old log files")
remove_old_logs(logs_by_dir, self._output_directory)
logging.debug("Writing per-dir HTML pages")
self.print_by_dir(self._output_directory, logs_by_dir)
total_packages = self.write_counts_summary()
self.create_package_summaries(logs_by_dir)
logging.debug("Writing section index page")
self.write_section_index_page(dirs, total_packages)
logging.debug("Writing stats pages for %s" % self._config.section)
self.write_state_pages()
def generate_output(self, output_directory, section_names):
# skip output generation for disabled sections
if int(self._config["max-reserved"]) == 0:
return
self._section_names = section_names
self._section_navigation = create_section_navigation(
self._section_names, self._config.section, self._doc_root)
self._output_directory = os.path.abspath(os.path.join(output_directory, self._config.section))
if not os.path.exists(self._output_directory):
os.makedirs(self._output_directory)
oldcwd = os.getcwd()
os.chdir(self._section_directory)
self.generate_html()
os.chdir(oldcwd)
def main():
setup_logging(logging.DEBUG, None)
if len(sys.argv) > 1:
print 'piuparts-report takes no command line parameters.'
sys.exit(1)
global_config = Config(section="global")
global_config.read(CONFIG_FILE)
if global_config["proxy"]:
os.environ["http_proxy"] = global_config["proxy"]
section_names = global_config["sections"].split()
master_directory = global_config["master-directory"]
output_directory = global_config["output-directory"]
doc_root = global_config["doc-root"].strip()
if not doc_root.startswith("/"):
doc_root = "/" + doc_root
if doc_root.endswith("/"):
doc_root = doc_root[:-1]
if os.path.exists(master_directory):
packagedb_cache = {}
write_file(os.path.join(output_directory, "sections.yaml"),
yaml.dump(section_names, default_flow_style=False))
for section_name in section_names:
try:
section = Section(section_name, master_directory, doc_root, packagedb_cache=packagedb_cache)
except MissingSection as e:
logging.error("Configuration Error in section '%s': %s" % (section_name, e))
else:
section.generate_output(output_directory=output_directory, section_names=section_names)
# static pages
logging.debug("Writing static pages")
for page in ("index", "bug_howto"):
tpl = os.path.join(output_directory, page+".tpl")
INDEX_BODY = "".join(read_file(tpl))
write_template_html(
os.path.join(output_directory, page+".html"),
INDEX_BODY,
{
"page_title": "About piuparts.debian.org and News",
"section_navigation": create_section_navigation(section_names, "sid", doc_root),
"doc_root": doc_root,
})
else:
logging.debug("Warning: %s does not exist!?! Creating it for you now." % master_directory)
os.makedirs(master_directory)
if __name__ == "__main__":
main()
# vi:set et ts=4 sw=4 :
piuparts-0.56/bug-templates/ 0000775 0000000 0000000 00000000000 12253565512 012762 5 ustar piuparts-0.56/bug-templates/fails_to_install_purge_install.mail 0000664 0000000 0000000 00000001263 12144650665 022111 0 ustar To: submit@bugs.debian.org
Subject: fails to install, purge, and install again
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to install,
remove+purge, and install again.
Like a plain failure on initial install this makes the package too buggy
for a release, thus the severity.
This is often a problem with the home directory of a system user:
the user is not deleted during purge (which is fine), but the home
directory is removed. Since the user already exists on the next
installation, adduser won't recreate the home.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_install_due_to_incorrect_dependencies_in_init.d_LSB_header.mail0000664 0000000 0000000 00000001044 12144650665 031236 0 ustar To: submit@bugs.debian.org
Subject: fails to install due to incorrect dependencies in init.d LSB header
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
User: initscripts-ng-devel@lists.alioth.debian.org
Usertags: incorrect-dependency
Hi,
during a test with piuparts I noticed your package failed to install due
to incorrect dependencies in the init.d LSB header. Some Debian notes
are available from at http://wiki.debian.org/LSBInitScripts
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_install.mail 0000664 0000000 0000000 00000000600 12144650665 017153 0 ustar To: submit@bugs.debian.org
Subject: fails to install
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to install. As
per definition of the release team this makes the package too buggy for
a release, thus the severity.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/unowned_files_in_usr_local_after_purge_policy_6.8.mail 0000664 0000000 0000000 00000001152 12144650665 025557 0 ustar To: submit@bugs.debian.org
Subject: unowned files in /usr/local after purge (policy 6.8)
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned files on
the system after purge, which is a violation of policy 6.8:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
From the attached log (scroll to the bottom...):
As putting files into /usr/local is also a violation of
http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.1.2
I'm setting the severity to serious.
cheers,
piuparts-0.56/bug-templates/usertag_piuparts.mail 0000664 0000000 0000000 00000000170 12144650665 017230 0 ustar To: control@bugs.debian.org
Subject: usertag piuparts
user debian-qa@lists.debian.org
usertag 123456 + piuparts
thanks
piuparts-0.56/bug-templates/doesnt_use_invoke-rc.d.mail 0000664 0000000 0000000 00000001121 12144650665 020173 0 ustar To: submit@bugs.debian.org
Subject: doesn't use invoke-rc.d
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package starts processes
where it shouldn't. This is very probably due to not using invoke-rc.d
as mandated by policy 9.3.3.2. This is seriously disturbing! ;-)
See http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.3.3
and /usr/share/doc/sysv-rc/README.invoke-rc.d.gz as well
as /usr/share/doc/sysv-rc/README.policy-rc.d.gz
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_lenny.mail 0000664 0000000 0000000 00000000560 12144650665 021371 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from lenny
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'lenny'.
It installed fine in 'lenny', then the upgrade to 'squeeze' fails.
From the attached log (scroll to the bottom...):
cheers,
././@LongLink 0000644 0000000 0000000 00000000156 00000000000 011605 L ustar root root piuparts-0.56/bug-templates/prompts_user_without_following_Debian_Configuration_Management_Specification.mail piuparts-0.56/bug-templates/prompts_user_without_following_Debian_Configuration_Management_Specifica0000664 0000000 0000000 00000003110 12144650665 031543 0 ustar To: submit@bugs.debian.org
Subject: prompts user without following Debian Configuration Management Specification
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package prompts the user
badly. Prompting in maintainer scripts must be done by communicating
through a program such as debconf which conforms to the Debian
Configuration Management Specification, version 2 or higher.
Refer to Debian Policy Manual section 3.9.1 (Prompting in maintainer
scripts) for details. Quoting from
http://www.debian.org/doc/debian-policy/ch-binary.html#s-maintscriptprompt
------------------------------------------------------------------------------
[ 3.9.1 Prompting in maintainer scripts ]
Package maintainer scripts may prompt the user if necessary. Prompting
must be done by communicating through a program, such as debconf, which
conforms to the Debian Configuration Management Specification, version 2
or higher.
Packages which are essential, or which are dependencies of essential
packages, may fall back on another prompting method if no such interface
is available when they are executed.
The Debian Configuration Management Specification is included in the
debconf_specification files in the debian-policy package.
------------------------------------------------------------------------------
From the attached log (scroll to the bottom...):
And as it's not possible to set two usertags at the same time, this also
should to be done:
User: lintian-maint@debian.org
Usertags: read-in-maintainer-script
cheers,
piuparts-0.56/bug-templates/fails_to_install_trying_to_overwrite_other_packages_files.mail 0000664 0000000 0000000 00000000774 12144650665 027614 0 ustar To: submit@bugs.debian.org
Subject: fails to install, trying to overwrite other packages files
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to install
because it tries to overwrite other packages files without declaring a
Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
cheers,
././@LongLink 0000644 0000000 0000000 00000000150 00000000000 011577 L ustar root root piuparts-0.56/bug-templates/fails_to_upgrade_from_lenny_to_squeeze_to_wheezy_-_trying_to_overwrite.mail piuparts-0.56/bug-templates/fails_to_upgrade_from_lenny_to_squeeze_to_wheezy_-_trying_to_overwrite.m0000664 0000000 0000000 00000001233 12144650665 031661 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade lenny -> squeeze -> wheezy - trying to overwrite ...
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'lenny' to 'squeeze' to 'wheezy'.
It installed fine in 'lenny', and upgraded to 'squeeze' successfully,
but then the upgrade to 'wheezy' failed because it tries to overwrite
other packages files without declaring a Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/unowned_directory_after_purge.mail 0000664 0000000 0000000 00000001506 12144650665 021761 0 ustar To: submit@bugs.debian.org
Subject: unowned directory after purge:
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned
directories on the system after purge, which is a violation of
policy 6.8:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
Filing this as important as having a piuparts clean archive is a release
goal since lenny.
The maintainer scripts create (and later remove) a file in that
directory. Manual directory removal may be not appropriate as this
directory is shared between several packages.
If the package would ship this as an empty directory, dpkg would take
care of the creation and removal (if it's empty).
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_lenny_to_squeeze_to_wheezy_to_jessie.mail 0000664 0000000 0000000 00000000750 12253565512 027773 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade lenny -> squeeze -> wheezy -> jessie
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'lenny' to 'squeeze' to 'wheezy' to 'jessie'.
It installed fine in 'lenny', and upgraded to 'squeeze' and 'wheezy'
successfully, but then the upgrade to 'jessie' failed.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/partial-upgrade-file-overwrite.mail 0000664 0000000 0000000 00000003042 12144650665 021652 0 ustar To: submit@bugs.debian.org
Subject: FOO: missing Breaks+Replaces: BAR (<< VBAR)
Package: FOO
Version: VFOO
Severity: serious
User: treinen@debian.org
Usertags: edos-file-overwrite
Architecture: amd64
Distribution: squeeze->wheezy (partial) upgrade
Hi,
automatic installation tests of packages that share a file and at the
same time do not conflict by their package dependency relationships has
detected the following problem:
This is a serious bug as it makes installation/upgrade fail, and
violates sections 7.6.1 and 10.1 of the policy.
As this problem can be demonstrated during partial upgrades from squeeze
to wheezy (but not within squeeze or wheezy itself), this indicates a
missing or insufficiently versioned Replaces+Breaks relationship.
But since this particular upgrade ordering is not forbidden by any
dependency relationship, it is possible that apt (or $PACKAGE_MANAGER)
will use this erroneus path on squeeze->wheezy upgrades.
Here is a list of files that are known to be shared by both packages
(according to the Contents files for squeeze and wheezy on amd64, which
may be slightly out of sync):
The following relationships are currently defined:
Package:
Conflicts: n/a
Breaks: n/a
Replaces: n/a
The following relationships should be added for a clean takeover of
these files
(http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces):
Package:
Breaks:
Replaces:
Cheers,
PS: for more information about the detection of file overwrite errors
of this kind see http://edos.debian.net/file-overwrites/.
piuparts-0.56/bug-templates/modifying_files_from_another_package.mail 0000664 0000000 0000000 00000000635 12144650665 023222 0 ustar To: submit@bugs.debian.org
Subject: modifying files from another package
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package modifies files from
another package in /usr. This is so wrong, I'm not even bothered to look
up the part of policy this violates ;-P
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/creates_system_user_in_home.mail 0000664 0000000 0000000 00000001764 12144650665 021427 0 ustar To: submit@bugs.debian.org
Subject: creates system user in /home
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package creates a system user
with a home directory in /home/$SYSUSER, which is a policy violation.
The FHS (which is part of the policy) states:
"/home : User home directories (optional)"
"/home is a fairly standard concept, but it is clearly a site-specific
filesystem. The setup will differ from host to host. Therefore, no
program should rely on this location."
http://www.debian.org/doc/packaging-manuals/fhs/fhs-2.3.html#HOMEUSERHOMEDIRECTORIES
System users are usually placed in /var/lib.
Possible problems that can arise in case /home is a remote file system:
* the local root user may not be allowed to write to /home
* $SYSUSER may have different UIDs on different hosts
For that particular piuparts test /home has been made unusable.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_purge_-_command_deluser_adduser_in_postrm_not_found.mail 0000664 0000000 0000000 00000002676 12144650665 030157 0 ustar To: submit@bugs.debian.org
Subject: fails to purge - command (deluser|adduser) in postrm not found
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to purge due
to a command not found. According to policy 7.2 you cannot rely on the
depends being available during purge, only the essential packages are
available for sure.
The fix should be easy: your package is using adduser or deluser from
the adduser package, which is only priority important. Using useradd or
userdel from the passwd package (priority required) should fix this
problem.
There is ongoing discussion how to handle system users on package
removal, see http://bugs.debian.org/621833
Consensus seems to be not to remove system users (to avoid reusing UIDs
which could grant access to the wrong files) but to "lock" them (where
"locking"/"unlocking" is not yet precisely defined). Until that has
been decided it should be sufficient to have the postrm script ignore
any errors from deluser:
deluser ... || true
Filing this as important because a.) it's a clear policy violation (to
not clean up at purge) b.) having a piuparts clean archive is a release
goal since lenny and c.) this package being piuparts buggy blocks
packages depending on it from being tested by piuparts (and thus
possibly the detection of more severe problems).
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/leaves_alternatives_after_purge.mail 0000664 0000000 0000000 00000003113 12144650665 022252 0 ustar To: submit@bugs.debian.org
Subject: leaves alternatives after purge
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned files on
the system after purge, which is a violation of policy 6.8:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
The leftover files are actually alternatives that were installed by the
package but have not been properly removed.
While there is ongoing discussion how to remove alternatives correctly
(see http://bugs.debian.org/71621 for details) the following strategy
should work for regular cases:
* 'postinst configure' always installs the alternative
* 'prerm remove' removes the alternative
* 'postrm remove' and 'postrm disappear' remove the alternative
In all other cases a maintainer script is invoked (e.g. upgrade,
deconfigure) the alternatives are not modified to preserve user
configuration.
Removing the alternative in 'prerm remove' avoids having a dangling link
once the actual file gets removed, but 'prerm remove' is not called in
all cases (e.g. unpacked but not configured packages or disappearing
packages) so the postrm must remove the alternative again
(update-alternatives gracefully handles removal of non-existing
alternatives).
Note that the arguments for adding and removing alternatives differ, for
removal it's 'update-alternatives --remove '.
Filing this as important as having a piuparts clean archive is a release
goal since lenny.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_install_due_to_insserv_rejecting_the_script_header.mail 0000664 0000000 0000000 00000000663 12144650665 030062 0 ustar To: submit@bugs.debian.org
Subject: fails to install due to insserv rejecting the script header
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to install due
to insserv rejecting the script header. Some notes are
available from at http://wiki.debian.org/LSBInitScripts
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/unowned_ls-R_file_in_usr_local_after_purge.mail 0000664 0000000 0000000 00000002540 12144650665 024321 0 ustar To: submit@bugs.debian.org
Subject: unowned file /usr/local/share/texmf/ls-R after purge (policy 6.8, 9.1.2)
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned files on
the system after purge, which is a violation of policy 6.8:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
As putting files into /usr/local is also a violation of
http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.1.2
I'm setting the severity to serious.
From the attached log (scroll to the bottom...):
0m53.8s ERROR: FAIL: Package purging left files on system:
/usr/local/share/texmf/ not owned
/usr/local/share/texmf/ls-R not owned
This problem is usually caused by running mktexlsr (or texhash) without
path arguments from a maintainer script.
The recommende solution is to switch to use dh_installtex and have this
generate most (or perhaps even all) of the maintainer scripts content.
Otherwise run mktexlsr with the tree as argument where the package
installs its files, which is usually
mktexlsr /usr/share/texmf
Please have a look at the Debian-TeX-Policy (in the tex-common package)
for the current practice of handling TeX packages. For further TeX
packaging related questions contact debian-tex-maint@lists.debian.org
cheers,
piuparts-0.56/bug-templates/copyright_file_missing_after_upgrade.mail 0000664 0000000 0000000 00000002215 12253565512 023256 0 ustar To: submit@bugs.debian.org
Subject: copyright file missing after upgrade (policy 12.5)
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
a test with piuparts revealed that your package misses the copyright
file after an upgrade, which is a violation of Policy 12.5:
http://www.debian.org/doc/debian-policy/ch-docs.html#s-copyrightfile
After the upgrade /usr/share/doc/$PACKAGE/ is just an empty directory.
This was observed on the following upgrade paths:
From the attached log (scroll to the bottom...):
Additional info may be available here:
http://wiki.debian.org/MissingCopyrightFile
Note that dpkg intentionally does not replace directories with symlinks
and vice versa, you need the maintainer scripts to do this.
See in particular the end of point 4 in
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-unpackphase
It is recommended to use the dpkg-maintscript-helper commands
'dir_to_symlink' and 'symlink_to_dir' (available since dpkg 1.17.2)
to perform the conversion, ideally using d/$PACKAGE.mainstscript.
See dpkg-maintscript-helper(1) and dh_installdeb(1) for details.
cheers,
piuparts-0.56/bug-templates/fails_to_install_remove_install.mail 0000664 0000000 0000000 00000001257 12144650665 022267 0 ustar To: submit@bugs.debian.org
Subject: fails to install, remove, and install again
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to install,
remove (but not purge), and install again.
Before the second installation the package is in config-files-remaining
state. The configuration is remaining from the last version that was
successfully configured - which is the same version that is going to be
installed again.
Like a plain failure on initial install this makes the package too buggy
for a release, thus the severity.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_squeeze.mail 0000664 0000000 0000000 00000000565 12144650665 021732 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from squeeze
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'squeeze'.
It installed fine in 'squeeze', then the upgrade to 'wheezy' fails.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/uninstallable_in_sid.mail 0000664 0000000 0000000 00000000447 12144650665 020020 0 ustar To: submit@bugs.debian.org
Subject: uninstallable in sid
Package:
Version:
Severity: grave
Justification: renders package unusable
Tags: sid
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package is no longer
installable in sid:
Cheers,
piuparts-0.56/bug-templates/fails_to_purge_-_command_in_postrm_not_found.mail 0000664 0000000 0000000 00000001433 12144650665 024713 0 ustar To: submit@bugs.debian.org
Subject: fails to purge - command in postrm not found
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to purge due
to a command not found. According to policy 7.2 you cannot rely on the
depends being available during purge, only the essential packages are
available for sure.
Filing this as important because a.) it's a clear policy violation (to
not clean up at purge) b.) having a piuparts clean archive is a release
goal since lenny and c.) this package being piuparts buggy blocks
packages depending on it from being tested by piuparts (and thus
possibly the detection of more severe problems).
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/unhandled_symlink_to_directory_conversion.mail 0000664 0000000 0000000 00000003201 12253565512 024365 0 ustar To: submit@bugs.debian.org
Subject: unhandled symlink to directory conversion: /usr/share/doc/PACKAGE
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
an upgrade test with piuparts revealed that your package installs files
over existing symlinks and possibly overwrites files owned by other
packages. This usually means an old version of the package shipped a
symlink but that was later replaced by a real (and non-empty)
directory. This kind of overwriting another package's files cannot be
detected by dpkg.
This was observed on the following upgrade paths:
For /usr/share/doc/PACKAGE this may not be problematic as long as both
packages are installed, ship byte-for-byte identical files and are
upgraded in lockstep. But once one of the involved packages gets
removed, the other one will lose its documentation files, too,
including the copyright file, which is a violation of Policy 12.5:
http://www.debian.org/doc/debian-policy/ch-docs.html#s-copyrightfile
For other overwritten locations anything interesting may happen.
Note that dpkg intentionally does not replace directories with symlinks
and vice versa, you need the maintainer scripts to do this.
See in particular the end of point 4 in
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-unpackphase
It is recommended to use the dpkg-maintscript-helper commands
'dir_to_symlink' and 'symlink_to_dir' (available since dpkg 1.17.2)
to perform the conversion, ideally using d/$PACKAGE.mainstscript.
See dpkg-maintscript-helper(1) and dh_installdeb(1) for details.
From the attached log (usually somewhere in the middle...):
cheers,
piuparts-0.56/bug-templates/removal_makes_files_disappear.mail 0000664 0000000 0000000 00000002124 12144650665 021667 0 ustar To: submit@bugs.debian.org
Subject: removal of $OFFENDER makes files disappear from $VICTIM
Package: $OFFENDER
Affects: $VICTIM
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts replaces-without-breaks
Hi,
during a test with piuparts and EDOS tools I noticed your package causes
removal of files that also belong to another package.
This is caused by using Replaces without corresponding Breaks.
The installation sequence to reproduce this problem is
apt-get install $VICTIM
# (1)
apt-get install $OFFENDER
apt-get remove $OFFENDER
# (2)
The list of installed files at points (1) and (2) should be identical,
but the following files have disappeared:
This is a serious bug violating policy 7.6, see
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
and also see the footnote that describes this incorrect behavior
http://www.debian.org/doc/debian-policy/footnotes.html#f53
The $OFFENDER package has the following relationships with $VICTIM:
Conflicts:
Breaks:
Replaces:
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_purge_-_command_ucf_in_postrm_not_found.mail 0000664 0000000 0000000 00000001647 12144650665 025557 0 ustar To: submit@bugs.debian.org
Subject: fails to purge - command ucf in postrm not found
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to purge due
to a command not found. According to policy 7.2 you cannot rely on the
depends being available during purge, only the essential packages are
available for sure.
Please see the manpages ucf(1), ucfr(1) and the example maintainer
scripts under /usr/share/doc/ucf/examples/ for correct usage of ucf.
Filing this as important because a.) it's a clear policy violation (to
not clean up at purge) b.) having a piuparts clean archive is a release
goal since lenny and c.) this package being piuparts buggy blocks
packages depending on it from being tested by piuparts (and thus
possibly the detection of more severe problems).
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_purge_due_to_incorrect_dependencies_in_init.d_LSB_header.mail 0000664 0000000 0000000 00000001042 12144650665 030710 0 ustar To: submit@bugs.debian.org
Subject: fails to purge due to incorrect dependencies in init.d LSB header
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
User: initscripts-ng-devel@lists.alioth.debian.org
Usertags: incorrect-dependency
Hi,
during a test with piuparts I noticed your package failed to purge due
to incorrect dependencies in the init.d LSB header. Some Debian notes
are available from at http://wiki.debian.org/LSBInitScripts
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_wheezy.mail 0000664 0000000 0000000 00000000564 12223270751 021553 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from 'wheezy'
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'wheezy'.
It installed fine in 'wheezy', then the upgrade to 'jessie' fails.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_lenny_to_squeeze_to_wheezy.mail 0000664 0000000 0000000 00000000705 12144650665 025732 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade lenny -> squeeze -> wheezy
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'lenny' to 'squeeze' to 'wheezy'.
It installed fine in 'lenny', and upgraded to 'squeeze' successfully,
but then the upgrade to 'wheezy' failed.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/owned_and_unowned_files_after_purge_policy_6.8_+_10.7.3.mail 0000664 0000000 0000000 00000001046 12144650665 026066 0 ustar To: submit@bugs.debian.org
Subject: owned and unowned files after purge (policy 6.8 + 10.7.3)
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left owned and
unowned files on the system after purge, which is a violation of
policy 6.8 and 10.7.3:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.3
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_remove.mail 0000664 0000000 0000000 00000000423 12144650665 017005 0 ustar To: submit@bugs.debian.org
Subject: fails to remove
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to remove.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/postinst_uses_usr_share_doc.mail 0000664 0000000 0000000 00000001363 12151074163 021445 0 ustar To: submit@bugs.debian.org
Subject: postinst uses /usr/share/doc content (Policy 12.3)
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
a test with piuparts revealed that your package uses files from
/usr/share/doc in its maintainer scripts which is a violation of
Policy 12.3: "Packages must not require the existence of any files in
/usr/share/doc/ in order to function."
http://www.debian.org/doc/debian-policy/ch-docs.html#s12.3
These files must be moved to /usr/share/$PACKAGE and may be symlinked
from /usr/share/doc/$PACKAGE.
This piuparts test prevents the installation of (most) files into
/usr/share/doc with 'dpkg --path-exclude=...'.
From the attached log (scroll to the bottom...):
Cheers,
piuparts-0.56/bug-templates/unowned_directories_in_usr_local_after_purge_policy_6.8_and_9.1.2.mail 0000664 0000000 0000000 00000001167 12144650665 030350 0 ustar To: submit@bugs.debian.org
Subject: unowned directories in /usr/local after purge (policy 6.8 and 9.1.2)
Package:
Version:
Severity: normal
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned files on
the system after purge, which is a violation of policy 6.8:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
From the attached log (scroll to the bottom...):
Leaving directories in /usr/local after purge is also a violation of a
should-directive in
http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.1.2
cheers,
piuparts-0.56/bug-templates/fails_to_install_remove_distupgrade_install.mail 0000664 0000000 0000000 00000001316 12144650665 024656 0 ustar To: submit@bugs.debian.org
Subject: fails to install, remove, distupgrade, and install again
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to install
(in 'CODENAME1'), remove (but not purge), distupgrade to 'CODENAME2',
and install again.
Before the second installation the package is in config-files-remaining
state. The configuration is remaining from the last version that was
successfully configured - which is from the previous release.
Like a plain failure on initial install this makes the package too buggy
for a release, thus the severity.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/leaves_diversions_after_upgrade.mail 0000664 0000000 0000000 00000001156 12144650665 022250 0 ustar To: submit@bugs.debian.org
Subject: leaves diversion after upgrade from squeeze
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to remove some
diversions after upgrading from squeeze and removing the package
afterwards.
Filing this as important as having a piuparts clean archive is a release
goal since lenny.
From the attached log (scroll to the bottom...):
The test did the following:
setup minimal squeeze chroot
install $package/squeeze
distupgrade wheezy
remove $package
purge $package
cheers,
piuparts-0.56/bug-templates/cronjob_exits_with_error_after_package_removal.mail 0000664 0000000 0000000 00000000542 12144650665 025327 0 ustar To: submit@bugs.debian.org
Subject: cronjob exits with error after package removal
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your packages cronjob exits with
error after the package has been removed.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/owned_after_purge_policy_6.8_+_10.7.3.mail 0000664 0000000 0000000 00000001010 12144650665 022312 0 ustar To: submit@bugs.debian.org
Subject: owned after purge (policy 6.8 + 10.7.3)
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left owned files on
the system after purge, which is a violation of policy 6.8 and 10.7.3:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.3
From the attached log (scroll to the bottom...):
cheers,
././@LongLink 0000644 0000000 0000000 00000000151 00000000000 011600 L ustar root root piuparts-0.56/bug-templates/prompting_due_to_modified_conffiles_which_were_not_modified_by_the_user.mail piuparts-0.56/bug-templates/prompting_due_to_modified_conffiles_which_were_not_modified_by_the_user.0000664 0000000 0000000 00000002251 12144650665 031550 0 ustar To: submit@bugs.debian.org
Subject: prompting due to modified conffiles which were not modified by the user
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed the piuparts
upgrade test because dpkg detected a conffile as being modified and then
prompted the user for an action. As there is no user input, this fails.
But this is not the real problem, the real problem is that this prompt
shows up in the first place, as there was nobody modifying this conffile
at all, the package has just been installed and upgraded...
This is a violation of policy 10.7.3, see
http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.3,
which says "[These scripts handling conffiles] must not ask unnecessary
questions (particularly during upgrades), and must otherwise be good
citizens."
http://wiki.debian.org/DpkgConffileHandling should help with figuring
out how to do this properly.
In http://lists.debian.org/debian-devel/2009/08/msg00675.html and
followups it has been agreed that these bugs are to be filed with
severity serious.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_-_trying_to_overwrite.mail 0000664 0000000 0000000 00000001527 12144650665 023565 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from 'DISTRO1' - trying to overwrite ...
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'DISTRO1'.
It installed fine in 'DISTRO1', then the upgrade to 'DISTRO2' fails
because it tries to overwrite files that are owned by other packages
without declaring a Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
This problem is exposed during the upgrade of the 'EXPOSER'
package, not by upgrading only the buggy package itself.
In order to have piuparts automatically track this problem, I'll mark
this bug as Affects/Found in the exposing package(s), too.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_testing_-_trying_to_overwrite.mail 0000664 0000000 0000000 00000001112 12144650665 026333 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from 'testing' - trying to overwrite ...
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'testing'.
It installed fine in 'testing', then the upgrade to 'sid' fails
because it tries to overwrite other packages files without declaring a
Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/cronjob_produces_output_after_package_removal.mail 0000664 0000000 0000000 00000000540 12144650665 025171 0 ustar To: submit@bugs.debian.org
Subject: cronjob produces output after package removal
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your packages cronjob produces
output after the package has been removed.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_wheezy_-_trying_to_overwrite.mail 0000664 0000000 0000000 00000001112 12223270751 026161 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from 'wheezy' - trying to overwrite ...
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'wheezy'.
It installed fine in 'wheezy', then the upgrade to 'jessie' fails
because it tries to overwrite other packages files without declaring a
Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/removes_files_installed_by_other_package.mail 0000664 0000000 0000000 00000002123 12144650665 024076 0 ustar To: submit@bugs.debian.org
Subject: removes files that were installed by another package:
Package:
Affects:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package removes files that
were installed by another package.
The removed files were already present before the package was installed,
they may have been shipped or created by a dependency.
This could be a violation of policy 10.7.4 ("Sharing configuration
files"), see
http://www.debian.org/doc/debian-policy/ch-files.html#s10.7.4
or policy chapter 6 ("Package maintainer scripts..."), see
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html
or policy 7.6 ("Overwriting files..."), see
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
If a directory is used by several packages, all should ship it as part
of the package (possibly empty, using $package.dirs to create it), and
no package should mkdir/rmdir it in the maintainer scripts as dpkg will
take care of this.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/installs_over_existing_symlink.mail 0000664 0000000 0000000 00000002652 12144650665 022202 0 ustar To: submit@bugs.debian.org
Subject: directory vs. symlink conflict:
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package installs files over
an existing symlink shipped or created by another package.
Your package ships:
but package CONFLICTOR ships:
Installing something over existing symlinks is considered bad practice.
See e.g. http://lists.debian.org/87ehlevcrf.fsf@windlord.stanford.edu
It may break in subtle ways and dpkg cannot detect this as a problem.
* Your package might silently overwrite files installed at the symlink
destination by other packages.
* If the package shipping the symlink decides to make the link point
somewhere else (or turn it into a real directory), the files owned
by your package "will be lost" somewhere in the filesystem.
* Depending on installation order the problematic path will be created
either as a symlink or a directory: the package installed first will
"win" and all others have "lost".
Note that dpkg intentionally does not replace directories with
symlinks and vice versa, see in particular the end of point 4 in
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-unpackphase
(Note: Adding Pre-Depends is *not* a solution.)
Please move the files shipped in your package to the "real" location.
From the attached log (usually somewhere in the middle...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_squeeze_-_trying_to_overwrite.mail 0000664 0000000 0000000 00000001113 12144650665 026340 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from squeeze - trying to overwrite ...
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'squeeze'.
It installed fine in 'squeeze', then the upgrade to 'wheezy' fails
because it tries to overwrite other packages files without declaring a
Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/modifies_conffiles.mail 0000664 0000000 0000000 00000003176 12144650665 017467 0 ustar To: submit@bugs.debian.org
Subject: modifies conffiles (policy 10.7.3):
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package modifies conffiles.
This is forbidden by the policy, see
http://www.debian.org/doc/debian-policy/ch-files.html#s-config-files
10.7.3: "[...] The easy way to achieve this behavior is to make the
configuration file a conffile. [...] This implies that the default
version will be part of the package distribution, and must not be
modified by the maintainer scripts during installation (or at any
other time)."
Note that once a package ships a modified version of that conffile,
dpkg will prompt the user for an action how to handle the upgrade of
this modified conffile (that was not modified by the user).
Further in 10.7.3: "[...] must not ask unnecessary questions
(particularly during upgrades) [...]"
If a configuration file is customized by a maintainer script after
having asked some debconf questions, it may not be marked as a
conffile. Instead a template could be installed in /usr/share and used
by the postinst script to fill in the custom values and create (or
update) the configuration file (preserving any user modifications!).
This file must be removed during postrm purge.
ucf(1) may help with these tasks.
See also http://wiki.debian.org/DpkgConffileHandling
In https://lists.debian.org/debian-devel/2012/09/msg00412.html and
followups it has been agreed that these bugs are to be filed with
severity serious.
debsums reports modification of the following files,
from the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_upgrade_from_sid_-_trying_to_overwrite.mail 0000664 0000000 0000000 00000001131 12144650665 025436 0 ustar To: submit@bugs.debian.org
Subject: fails to upgrade from 'sid' - trying to overwrite ...
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'sid' to 'experimental'.
It installed fine in 'sid', then the upgrade to 'experimental' fails
because it tries to overwrite other packages files without declaring a
Breaks+Replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/package_removed_processes_still_running.mail 0000664 0000000 0000000 00000000774 12144650665 024012 0 ustar To: submit@bugs.debian.org
Subject: package removed, processes still running
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left processes
running after the package has been removed and/or purged.
In http://lists.debian.org/debian-devel/2009/08/msg00182.html and
followups it has been agreed that these bugs are to be filed with
severity serious.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/unowned_files_after_purge_policy_6.8_violating_FHS_policy_9.1_too.mail 0000664 0000000 0000000 00000001173 12144650665 030434 0 ustar To: submit@bugs.debian.org
Subject: unowned files after purge (policy 6.8) violating FHS (policy 9.1) too
Package:
Version:
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned files on
the system after purge, which is a violation of policy 6.8:
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
From the attached log (scroll to the bottom...):
As putting files into /usr/local is also a violation of
http://www.debian.org/doc/debian-policy/ch-opersys.html#s9.1.2
I'm setting the severity to serious.
cheers,
piuparts-0.56/bug-templates/unowned_files_after_purge_policy_6.8_and_10.8.mail 0000664 0000000 0000000 00000001055 12144650665 024320 0 ustar To: submit@bugs.debian.org
Subject: unowned files after purge (policy 6.8, 10.8)
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package left unowned files on
the system after purge, which is a violation of policy 6.8 (or 10.8):
http://www.debian.org/doc/debian-policy/ch-maintainerscripts.html#s-removedetails
Filing this as important as having a piuparts clean archive is a release
goal since lenny.
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/bug-templates/fails_to_purge.mail 0000664 0000000 0000000 00000001342 12144650665 016633 0 ustar To: submit@bugs.debian.org
Subject: fails to purge
Package:
Version:
Severity: important
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package failed to purge.
According to policy 7.2 you cannot rely on the depends being available
during purge, only the essential packages are available for sure.
Filing this as important because a.) it's a clear policy violation (to
not clean up at purge) b.) having a piuparts clean archive is a release
goal since lenny and c.) this package being piuparts buggy blocks
packages depending on it from being tested by piuparts (and thus
possibly the detection of more severe problems).
From the attached log (scroll to the bottom...):
cheers,
piuparts-0.56/custom-scripts/ 0000775 0000000 0000000 00000000000 12151074163 013202 5 ustar piuparts-0.56/custom-scripts/scripts/ 0000775 0000000 0000000 00000000000 12253565512 014677 5 ustar piuparts-0.56/custom-scripts/scripts/post_setup_forbid_home 0000775 0000000 0000000 00000000552 12175701020 021356 0 ustar #!/bin/sh
set -e
case ${PIUPARTS_OBJECTS%%=*} in
dpkg)
# skip while creating the tarball
exit 0
;;
esac
case $PIUPARTS_DISTRIBUTION in
lenny|squeeze|squeeze-proposed)
exit 0
;;
esac
if [ -d /home ]; then
echo "Disabling /home"
mv /home /home.orig
echo "This is a dummy file to prevent creating directories in /home" > /home
chmod 000 /home
fi
piuparts-0.56/custom-scripts/scripts/pre_remove_40_find_missing_md5sums 0000775 0000000 0000000 00000002306 12175701020 023467 0 ustar #!/bin/sh
set -e
# skip the md5sum check if /usr/share/doc is pruned
test ! -f /etc/dpkg/dpkg.cfg.d/piuparts-path-exclude || exit 0
for pkg in ${PIUPARTS_OBJECTS%%=*}
do
# skip check if the package is not installed
dpkg-query -s "$pkg" >/dev/null 2>&1 || continue
status="$(dpkg-query -W -f '${Status}' $pkg)"
test "$status" != "unknown ok not-installed" || continue
test "$status" != "deinstall ok config-files" || continue
md5file="/var/lib/dpkg/info/$pkg.md5sums"
test -f "$md5file" || md5file="/var/lib/dpkg/info/$pkg:$(dpkg --print-architecture).md5sums"
if [ ! -f "$md5file" ]; then
echo "MD5SUM FILE NOT FOUND FOR $pkg"
continue
fi
f1=/var/run/f1.$$
sed -r 's%^[0-9a-f]{32} %/%' "$md5file" | sort >$f1
f2=/var/run/f2.$$
>$f2
dpkg -L "$pkg" | sort | \
while read f ; do
if [ -d "$f" ]; then
: # ignore directories
elif [ -L "$f" ]; then
: # ignore links
elif [ -z "${f%%/etc/*}" ]; then
: # ignore files in /etc - probably conffiles
elif [ ! -e "$f" ]; then
echo "${pkg}: MISSING OBJECT $f"
else
echo "$f" >> $f2
fi
done
comm -13 $f1 $f2 | sed "s/^/${pkg}: FILE WITHOUT MD5SUM /"
comm -23 $f1 $f2 | sed "s/^/${pkg}: MD5SUM WITHOUT FILE /"
rm -f $f1 $f2
done
piuparts-0.56/custom-scripts/scripts/pre_remove_50_find_bad_permissions 0000775 0000000 0000000 00000004257 12253565512 023545 0 ustar #!/bin/sh
set -e
if [ "$PIUPARTS_DISTRIBUTION" = "squeeze" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze/updates" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze-updates" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze-proposed-updates" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze-proposed" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
case ${PIUPARTS_OBJECTS%%=*} in
citadel-server|citadel-dbg|citadel-mta|citadel-suite|bcron-run|capisuite|debbugs|raccess4vbox3|smartlist|sxid)
#WORKSAROUND #684964: citadel-server: world writable config file: /etc/citadel/netconfigs/7
for file in /etc/citadel/netconfigs/7 /etc/citadel/refcount_adjustments.dat /etc/citadel/citadel.control
do
test ! -f "$file" || chmod -c o-w "$file"
done
;;
linpopup)
# package removed in squeeze
for file in /var/lib/linpopup/messages.dat
do
test ! -f "$file" || chmod -c o-w "$file"
done
;;
esac
fi
if [ "$PIUPARTS_DISTRIBUTION" = "wheezy" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "wheezy/updates" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "wheezy-updates" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "wheezy-proposed-updates" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "wheezy-proposed" ]; then
case ${PIUPARTS_OBJECTS%%=*} in
citadel-server|citadel-dbg|citadel-mta|citadel-suite|bcron|bcron-run|capisuite|debbugs|exmh|nmh|raccess4vbox3|smartlist|xlbiff)
#WORKSAROUND #684964: citadel-server: world writable config file: /etc/citadel/netconfigs/7
for file in /etc/citadel/netconfigs/7 /etc/citadel/refcount_adjustments.dat /var/lib/citadel/data/refcount_adjustments.dat
do
test ! -f "$file" || chmod -c o-w "$file"
done
;;
esac
fi
case ${PIUPARTS_OBJECTS%%=*} in
gpe-tetris|gpe)
#WORKSAROUND #684178: gpe-tetris: creates world writable directory /var/games/gpe
# package removed after wheezy
for file in /var/games/gpe/gpe-tetris.dat
do
test ! -f "$file" || chmod -c o-w "$file"
done
for dir in /var/games/gpe
do
test ! -d "$dir" || chmod -c o-w "$dir"
done
;;
esac
# find world writables without sticky bit
BADPERMS=$(find / -mount ! -type l ! -type c ! -type p ! -type s -perm -o+w ! -perm -1000)
if [ -n "$BADPERMS" ]; then
echo "ERROR: BAD PERMISSIONS"
ls -lad $BADPERMS
exit 1
fi
piuparts-0.56/custom-scripts/scripts/post_distupgrade_exceptions 0000775 0000000 0000000 00000001136 12253565512 022447 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
if [ "$PIUPARTS_DISTRIBUTION" = "wheezy" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "wheezy-proposed" ] ; then
# libdb4.8/squeeze is part of the minimal squeeze chroot and
# remains installed after distupgrade even if it no longer
# exists in wheezy
# db4.8-util/wheezy Conflicts/Replaces libdb4.8, so ensure it
# gets removed from the reference chroot, too
case ${PIUPARTS_OBJECTS%%=*} in
db4.8-util|cyrus-*-2.2|libcyrus-imap-perl22|sa-learn-cyrus)
log_debug
dpkg --purge libdb4.8
;;
esac
fi
piuparts-0.56/custom-scripts/scripts/post_distupgrade_hack_debsums 0000775 0000000 0000000 00000000456 12144650665 022725 0 ustar #!/bin/sh
set -e
# http://bugs.debian.org/687611
if [ -f /usr/share/keyrings/debian-archive-removed-keys.gpg~ ]; then
echo "FIXING /usr/share/keyrings/debian-archive-removed-keys.gpg~"
mv -v /usr/share/keyrings/debian-archive-removed-keys.gpg~ /usr/share/keyrings/debian-archive-removed-keys.gpg
fi
piuparts-0.56/custom-scripts/scripts/pre_remove_exceptions 0000775 0000000 0000000 00000001137 12253565512 021233 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
#
# deal with exceptions:
#
case ${PIUPARTS_OBJECTS%%=*} in
isdnlog|isdnutils)
#WORKSAROUND #431855: fails with "There are still files in /etc/isdn/ that you may want to remove manually."
if [ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
log_debug
rm -fv /etc/isdn/*
fi
;;
esac
# Allow removal of the kernel running on the host from the chroot.
UNAME_R="$(uname -r)"
echo "linux-image-$UNAME_R linux-image-$UNAME_R/prerm/removing-running-kernel-$UNAME_R boolean false" | debconf-set-selections
piuparts-0.56/custom-scripts/scripts/pre_install_foreign_architecture_i386 0000775 0000000 0000000 00000000735 12253565512 024172 0 ustar #!/bin/sh
set -e
test "$PIUPARTS_PHASE" = "install" || exit 0
case "$PIUPARTS_DISTRIBUTION" in
lenny*|squeeze*) exit 0 ;;
esac
test "$(dpkg --print-architecture)" = "amd64" || exit 0
case ${PIUPARTS_OBJECTS%%=*} in
ia32-libs|ia32-libs-gtk) ;;
libwine-unstable|libwine-*-unstable|wine|wine-unstable) ;;
boinc-nvidia-cuda|teamspeak-client) ;;
*) exit 0 ;;
esac
echo "Enabling foreign architecture i386 for $PIUPARTS_OBJECTS"
dpkg --add-architecture i386
apt-get update
piuparts-0.56/custom-scripts/scripts/pre_remove_50_find_missing_copyright 0000775 0000000 0000000 00000002370 12253565512 024117 0 ustar #!/bin/sh
set -e
failed=
for pkg in ${PIUPARTS_OBJECTS%%=*}
do
# ignore failures for some old packages with many rdepends
ignore=
case "${pkg}_${PIUPARTS_DISTRIBUTION}" in
gij_lenny) ignore=1 ;;
gnumeric-common_lenny) ignore=1 ;; #554201
libuim6_lenny) ignore=1 ;; #554204
libuim-data_lenny) ignore=1 ;; #554200
mozilla-plugin-vlc_lenny) ignore=1 ;; #687657
postgresql-8.3-plsh_lenny) ignore=1 ;; # removed after lenny
vlc_lenny) ignore=1 ;; #687657
cdd-common_squeeze) ignore=1 ;; #692946
libfbclient2_squeeze) ignore=1 ;; #692948
libcucul0_wheezy) ignore=1 ;; # removed
esac
# skip check if the package is not installed
dpkg-query -s "$pkg" >/dev/null 2>&1 || continue
status="$(dpkg-query -W -f '${Status}' $pkg)"
test "$status" != "unknown ok not-installed" || continue
test "$status" != "deinstall ok config-files" || continue
docdir="/usr/share/doc/$pkg"
copyright="$docdir/copyright"
if [ ! -f "$copyright" ]
then
if [ -n "$ignore" ]; then
echo "ignoring failure of $pkg on $PIUPARTS_DISTRIBUTION"
else
failed="$failed $copyright"
fi
echo "MISSING COPYRIGHT FILE: $copyright"
echo "# ls -lad $docdir"
ls -lad "$docdir" || true
echo "# ls -la $docdir/"
ls -la "$docdir/" || true
fi
done
piuparts-0.56/custom-scripts/scripts/pre_test_root_password 0000775 0000000 0000000 00000000341 12144650665 021440 0 ustar #!/bin/sh
set -e
# sudo refuses removal if no root password is set, so set one
# do this unconditionally, as there are quite some packages depending on sudo
# (and since its harmless and fast)
yes "yes" 2>/dev/null | passwd
piuparts-0.56/custom-scripts/scripts/post_distupgrade_squeeze-backports 0000775 0000000 0000000 00000000622 12145710343 023726 0 ustar #!/bin/sh
set -e
test "$PIUPARTS_DISTRIBUTION" = "squeeze-backports" || exit 0
# apt-get -t squeeze-backports dist-upgrade may pull in too many
# packages that are not co-installable in squeeze-backports
# so maintain a list of packages in the sid base system
# that are in squeeze-backports, too, and don't cause problems
PKGS=""
PKGS="$PKGS insserv"
apt-get -y -t squeeze-backports install $PKGS
piuparts-0.56/custom-scripts/scripts/post_setup_experimental 0000777 0000000 0000000 00000000000 12144650665 027662 2post_distupgrade_experimental ustar piuparts-0.56/custom-scripts/scripts/pre_distupgrade_exceptions 0000775 0000000 0000000 00000001474 12145710343 022247 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "squeeze" ]; then
case ${PIUPARTS_OBJECTS%%=*} in
crm114)
#562946
log_debug
echo "crm114 crm114/forceupgrade boolean true" | debconf-set-selections
;;
esac
fi
if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "wheezy" ]; then
# dpkg 1.16 does not like the bad cnews version number cr.g7-40.4
# cnews was removed after lenny
case ${PIUPARTS_OBJECTS%%=*} in
cnews)
log_debug
dpkg --purge cnews
;;
esac
# WORKSAROUND #655969: lirc: prompting due to modified conffiles which where not modified by the user: /etc/lirc/hardware.conf
if [ -f /etc/lirc/hardware.conf ]; then
log_debug
sed -i '/^DRIVER=/s/.*/DRIVER="UNCONFIGURED"/' /etc/lirc/hardware.conf
fi
fi
piuparts-0.56/custom-scripts/scripts/pre_distupgrade_zz_upgrade_early 0000775 0000000 0000000 00000001151 12253565512 023432 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
# packages to upgrade early
EARLY=
if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "squeeze" ]; then
if dpkg-query -s "octave3.0" >/dev/null 2>&1
then
#696377
EARLY="$EARLY libblas3gf liblapack3gf"
fi
case ${PIUPARTS_OBJECTS%%=*} in
libapt-rpm-pkg-dev)
# libapt-rpm-pkg-dev no longer exists in squeeze
# causing some packages to be "kept back"
EARLY="$EARLY libreadline5-dev"
;;
esac
fi
if [ -n "$EARLY" ]; then
log_debug
echo "Upgrading early: $EARLY"
apt-get update
apt-get -y install $EARLY
fi
piuparts-0.56/custom-scripts/scripts/pre_install_exceptions 0000775 0000000 0000000 00000013055 12253565512 021406 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
#
# deal with packages depending on exceptions:
#
case ${PIUPARTS_OBJECTS%%=*} in
ltsp-client|education-thin-client)
log_debug
PIUPARTS_OBJECTS=ltsp-client-core
;;
upstart-dconf-bridge|upstart-monitor)
log_debug
PIUPARTS_OBJECTS=upstart
;;
netscript-2.4-upstart) log_debug
PIUPARTS_OBJECTS=upstart
;;
live-config-upstart) log_debug
PIUPARTS_OBJECTS=upstart
;;
clvm|dtc-xen|ganeti|ganeti2|mylvmbackup|redhat-cluster-suite|libvirt0|libcollectdclient0|liblinux-lvm-perl|autopkgtest-xenlvm|collectd-dbg|collectd|cman|libsys-virt-perl|libvirt-dev|libvirt-ocaml|libvirt-ruby1.8|libvirt0-dbg|python-libvirt|virt-top|virt-viewer|xenwatch|gfs-tools|gfs2-tools|rgmanager|virtinst|collectd-utils|libcollectdclient-dev|libvirt-ocaml-dev|libvirt-ruby|mozilla-virt-viewer|munin-libvirt-plugins)
# skip mini-buildd-bld here due to resource violation, see below
log_debug
PIUPARTS_OBJECTS=lvm2
;;
esac
#
# deal with exceptions:
#
case ${PIUPARTS_OBJECTS%%=*} in
fai-nfsroot) log_debug
# fai-nfsroot refuses installation unless this file exist
touch /.THIS_IS_THE_FAI_NFSROOT
;;
ltsp-client-core) log_debug
# ltsp-client-core refuses installation unless this file exist
touch /etc/ltsp_chroot
;;
upstart) log_debug
# force installation and removal of essential package sysvinit
yes 'Yes, do as I say!' | apt-get -y --force-yes install upstart
;;
file-rc) log_debug
# force installation and removal of essential package sysv-rc
yes 'Yes, do as I say!' | apt-get -y --force-yes install file-rc
;;
systemd-sysv) log_debug
# force installation and removal of essential package sysvinit
yes 'Yes, do as I say!' | apt-get -y --force-yes install systemd-sysv
;;
lvm2)
if [ "$PIUPARTS_PHASE" = "install" ] && ( \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze-proposed" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze-backports" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "lenny" ] )
then
# work around lvm2 bug http://bugs.debian.org/603036 which is squeeze-ignore
log_debug
apt-get -y install udev
fi
;;
bugzilla3)
# checksetup.pl goes into infinite loop asking for them
log_debug
echo "bugzilla3 bugzilla3/bugzilla_admin_name string bz@local.host" | debconf-set-selections
echo "bugzilla3 bugzilla3/bugzilla_admin_real_name string Bz" | debconf-set-selections
echo "bugzilla3 bugzilla3/bugzilla_admin_pwd password bzbzbzbz" | debconf-set-selections
;;
esac
if [ "$PIUPARTS_DISTRIBUTION" = "wheezy-backports" ]; then
case ${PIUPARTS_OBJECTS%%=*} in
libreoffice-style-*)
log_debug
apt-get -y -t $PIUPARTS_DISTRIBUTION install ${PIUPARTS_OBJECTS%%=*} libreoffice-common-
;;
esac
fi
if [ "$PIUPARTS_DISTRIBUTION" = "squeeze-backports" ]; then
case ${PIUPARTS_OBJECTS%%=*} in
ekeyd)
log_debug
apt-get -y install udev
;;
esac
fi
if [ "$PIUPARTS_DISTRIBUTION" = "squeeze" ]; then
case ${PIUPARTS_OBJECTS%%=*} in
bootcd-ia64)
if [ "$PIUPARTS_TEST" = "distupgrade" ] && [ "$(uname -m)" = "x86_64" ]; then
#622690: bootcd-ia64 has a Pre-Depends/Depends cycle that apt cannot resolve
log_debug
apt-get -y install bootcd-i386
fi
;;
mini-buildd-bld|mini-buildd-rep)
#632955, #656746 - time limit exceeded during install
log_debug
echo "*** ABORT - Installation would deadlock ***"
exit 1
;;
esac
fi
if [ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
# install undeclared dependencies
case ${PIUPARTS_OBJECTS%%=*} in
clamav-getfiles)
#603082
log_debug
apt-get -y install curl
;;
cyrus-*-2.2|sa-learn-cyrus)
#694254 db4.2-util exists in lenny only and is needed for upgrades to wheezy
log_debug
apt-get -y install db4.2-util
;;
debian-edu-artwork-usplash)
# postinst fails without update-initramfs
log_debug
apt-get -y install initramfs-tools
;;
gforge-shell-postgresql|gforge-plugin-scmcvs|gforge-plugin-scmsvn)
#604218
log_debug
apt-get -y install libcap2
;;
otrs2)
#561889
log_debug
apt-get -y install libapache2-mod-perl2
;;
sdic-gene95)
#478592
log_debug
apt-get -y install bzip2 wget
;;
tftpd-hpa)
#522780
log_debug
apt-get -y install update-inetd
;;
ttf-beteckna)
#502707
log_debug
apt-get -y install defoma
;;
esac
# prefer inn over cnews (bad version number cr.g7-40.4)
case ${PIUPARTS_OBJECTS%%=*} in
newsx|post-faq)
log_debug
apt-get -y install inn
;;
esac
# work around postinst failures
case ${PIUPARTS_OBJECTS%%=*} in
ion3)
# annoying debconf prompt
log_debug
echo "ion3 ion3/acknowledge-maybe-outdated boolean true" | debconf-set-selections
echo "ion3 ion3/acknowledge-outdated boolean true" | debconf-set-selections
;;
ipppd|ibod|isdnutils)
#542156
log_debug
test -e /dev/MAKEDEV || ln -sfv /sbin/MAKEDEV /dev/MAKEDEV
;;
esac
# allow lenny kernel image installation in chroot
for flavor in amd64 openvz-amd64 vserver-amd64 xen-amd64 486 686 686-bigmem openvz-686 vserver-686 vserver-686-bigmem xen-686
do
echo "linux-image-2.6.26-2-$flavor linux-image-2.6.26-2-$flavor/preinst/bootloader-initrd-2.6.26-2-$flavor boolean false"
echo "linux-image-2.6.26-2-$flavor linux-image-2.6.26-2-$flavor/preinst/lilo-initrd-2.6.26-2-$flavor boolean false"
done | debconf-set-selections
# deterministic /bin/sh on upgrades
echo "dash dash/sh boolean true" | debconf-set-selections
fi
piuparts-0.56/custom-scripts/scripts/post_distupgrade_experimental 0000775 0000000 0000000 00000000672 12253565512 022767 0 ustar #!/bin/sh
set -e
test "$PIUPARTS_DISTRIBUTION" = "experimental" || exit 0
# apt-get -t experimental dist-upgrade may pull in too many
# packages that are not co-installable in experimental
# so maintain a list of packages in the sid base system
# that are in experimental, too, and don't cause problems
PKGS=""
PKGS="$PKGS libapt-pkg4.12"
PKGS="$PKGS apt"
PKGS="$PKGS libc6"
PKGS="$PKGS libc-bin"
apt-get -y -t experimental install $PKGS
piuparts-0.56/custom-scripts/scripts/pre_distupgrade_zz_database-server 0000775 0000000 0000000 00000001522 12253565512 023661 0 ustar #!/bin/sh
set -e
CANDIDATES=""
CANDIDATES="$CANDIDATES mysql-server"
CANDIDATES="$CANDIDATES mysql-server-5.5"
CANDIDATES="$CANDIDATES postgresql"
CANDIDATES="$CANDIDATES postgresql-8.4"
CANDIDATES="$CANDIDATES postgresql-9.1"
PACKAGES=""
# early upgrade runs into even more trouble for some packages ...
case ${PIUPARTS_OBJECTS%%=*} in
med-practice|audiolink)
CANDIDATES=""
;;
redmine)
CANDIDATES=""
;;
esac
for pkg in $CANDIDATES
do
# skip if the package is not installed
dpkg-query -s "$pkg" >/dev/null 2>&1 || continue
status="$(dpkg-query -W -f '${Status}' $pkg)"
test "$status" != "unknown ok not-installed" || continue
test "$status" != "deinstall ok config-files" || continue
PACKAGES="$PACKAGES $pkg"
done
if [ -n "$PACKAGES" ]
then
echo "Upgrading ${PACKAGES# } early"
apt-get update
apt-get -y install $PACKAGES
fi
piuparts-0.56/custom-scripts/scripts/pre_install_database-server 0000775 0000000 0000000 00000012551 12253565512 022275 0 ustar #!/bin/sh
set -e
# Install a database server for packages that require one during postinst
# but only recommend or suggest (or maybe not even this) the server
# package to allow for remote db servers.
MYSQL=
POSTGRESQL=
SQLITE3=
case ${PIUPARTS_OBJECTS%%=*} in
acidbase) MYSQL=yes ;;
auth2db) MYSQL=yes ;;
auth2db-common) MYSQL=yes ;;
auth2db-frontend) MYSQL=yes ;;
b2evolution) MYSQL=yes ;;
bacula-director-mysql) MYSQL=yes ;;
bacula-director-mysql-dbg) MYSQL=yes ;;
bacula-director-pgsql) POSTGRESQL=yes ;;
bacula-director-pgsql-dbg) POSTGRESQL=yes ;;
bandwidthd-pgsql) POSTGRESQL=yes ;;
bley) POSTGRESQL=yes ;;
blootbot) MYSQL=yes ;;
buddycloud-server) POSTGRESQL=yes ;;
bugzilla3) MYSQL=yes ;;
cacti) MYSQL=yes ;;
cacti-cactid) MYSQL=yes ;;
cacti-spine) MYSQL=yes ;;
chado-utils) POSTGRESQL=yes ;;
convirt) MYSQL=yes ;;
docbookwiki) MYSQL=yes ;;
dotclear) MYSQL=yes ;;
dotlrn) POSTGRESQL=yes ;;
drupal6) MYSQL=yes ;;
drupal6-mod-*) MYSQL=yes ;;
drupal6-thm-*) MYSQL=yes ;;
drupal6-trans-ru) MYSQL=yes ;;
drupal7) MYSQL=yes ;;
drupal7-mod-libraries) MYSQL=yes ;;
frontaccounting) MYSQL=yes ;;
fossology-agents) POSTGRESQL=yes ;;
fusionforge-plugin-blocks) POSTGRESQL=yes ;;
fusionforge-plugin-extsubproj) POSTGRESQL=yes ;;
fusionforge-plugin-gravatar) POSTGRESQL=yes ;;
fusionforge-plugin-hudson) POSTGRESQL=yes ;;
fusionforge-plugin-mediawiki) POSTGRESQL=yes ;;
fusionforge-plugin-moinmoin) POSTGRESQL=yes ;;
fusionforge-plugin-projectlabels) POSTGRESQL=yes ;;
fusionforge-plugin-scmarch) POSTGRESQL=yes ;;
fusionforge-plugin-scmcvs) POSTGRESQL=yes ;;
fusionforge-plugin-scmdarcs) POSTGRESQL=yes ;;
fusionforge-standard) POSTGRESQL=yes ;;
fusioninventory-for-glpi) MYSQL=yes ;;
gforge-lists-mailman) POSTGRESQL=yes ;;
gforge-plugin-extratabs) POSTGRESQL=yes ;;
gforge-plugin-mediawiki) POSTGRESQL=yes ;;
gforge-plugin-scmarch) POSTGRESQL=yes ;;
gforge-plugin-scmcvs) POSTGRESQL=yes ;;
gforge-plugin-scmdarcs) POSTGRESQL=yes ;;
gforge-plugin-scmgit) POSTGRESQL=yes ;;
gforge-plugin-scmhg) POSTGRESQL=yes ;;
gforge-web-apache2) POSTGRESQL=yes ;;
gforge-web-apache2-vhosts) POSTGRESQL=yes ;;
glance) SQLITE3=yes ;;
glance-api) SQLITE3=yes ;;
glance-common) SQLITE3=yes ;;
glance-registry) SQLITE3=yes ;;
glpi) MYSQL=yes ;;
grr) MYSQL=yes ;;
heat-api) SQLITE3=yes ;;
heat-api-cfn) SQLITE3=yes ;;
heat-api-cloudwatch) SQLITE3=yes ;;
heat-common) SQLITE3=yes ;;
heat-engine) SQLITE3=yes ;;
icinga-idoutils) POSTGRESQL=yes ;;
icinga-phpapi) MYSQL=yes ;;
icinga-web) POSTGRESQL=yes ;;
icinga-web-pnp) POSTGRESQL=yes ;;
jffnms) MYSQL=yes ;;
letodms) MYSQL=yes ;;
libchado-perl) POSTGRESQL=yes ;;
libdspam7-drv-mysql) MYSQL=yes ;;
libdspam7-drv-pgsql) POSTGRESQL=yes ;;
mantis) MYSQL=yes ;;
moodle) POSTGRESQL=yes ; MYSQL=yes ;;
moodle-book) POSTGRESQL=yes ;;
moodle-debian-edu-*) POSTGRESQL=yes ;;
movabletype-opensource) SQLITE3=yes ;;
mtop) MYSQL=yes ;;
mythtv-database) MYSQL=yes ;;
nagvis) MYSQL=yes ;;
ndoutils-common) MYSQL=yes ;;
ndoutils-nagios3-mysql) MYSQL=yes ;;
nginx-naxsi-ui) MYSQL=yes ;;
ocsinventory-reports) MYSQL=yes ;;
ocsinventory-server) MYSQL=yes ;;
openacs) POSTGRESQL=yes ;;
otrs) POSTGRESQL=yes ;;
otrs2) POSTGRESQL=yes ; MYSQL=yes ;;
pdns-backend-mysql) MYSQL=yes ;;
pdns-backend-pgsql) POSTGRESQL=yes ;;
phpbb3) MYSQL=yes ;;
phpbb3-l10n) MYSQL=yes ;;
phpgacl) MYSQL=yes ;;
phpmyadmin) MYSQL=yes ;;
phpwiki) MYSQL=yes ;;
pinba-engine-mysql-5.5) MYSQL=yes ;;
piwigo) MYSQL=yes ;;
pnopaste) MYSQL=yes ;;
poker-web) MYSQL=yes ;;
postfix-policyd) MYSQL=yes ;;
postfixadmin) MYSQL=yes ;;
prelude-manager) MYSQL=yes ;;
prewikka) MYSQL=yes ;;
pybit-web) POSTGRESQL=yes ;;
redmine) MYSQL=yes ;;
redmine-mysql) MYSQL=yes ;;
redmine-pgsql) POSTGRESQL=yes ;;
redmine-plugin-botsfilter) MYSQL=yes ;;
redmine-plugin-recaptcha) MYSQL=yes ;;
roundcube) MYSQL=yes ;;
roundcube-core) MYSQL=yes ;;
roundcube-plugins) MYSQL=yes ;;
roundcube-plugins-extra) MYSQL=yes ;;
rsyslog-mysql) MYSQL=yes ;;
rsyslog-pgsql) POSTGRESQL=yes ;;
scuttle) MYSQL=yes ;;
semanticscuttle) MYSQL=yes ;;
serendipity) MYSQL=yes ;;
simba) MYSQL=yes ;;
spotweb) MYSQL=yes ;;
sshproxy-backend-mysql) MYSQL=yes ;;
steam) MYSQL=yes ;;
sympa) MYSQL=yes ;;
tango-accesscontrol) MYSQL=yes ;;
tango-accesscontrol-dbg) MYSQL=yes ;;
tango-db) MYSQL=yes ;;
tango-db-dbg) MYSQL=yes ;;
textpattern) MYSQL=yes ;;
torrentflux) MYSQL=yes ;;
tt-rss) POSTGRESQL=yes ;;
typo3-dummy) MYSQL=yes ;;
ukolovnik) MYSQL=yes ;;
webcalendar) MYSQL=yes ;;
webissues-server) MYSQL=yes ;;
websimba) MYSQL=yes ;;
wims-moodle) POSTGRESQL=yes ;;
zabbix-frontend-php) MYSQL=yes ;;
zabbix-proxy-mysql) MYSQL=yes ;;
zabbix-proxy-pgsql) POSTGRESQL=yes ;;
zabbix-server-mysql) MYSQL=yes ;;
zabbix-server-pgsql) POSTGRESQL=yes ;;
esac
if [ "$MYSQL" = "yes" ]; then
echo "Installing mysql-server..."
apt-get -y install mysql-server
fi
if [ "$POSTGRESQL" = "yes" ]; then
echo "Installing postgresql..."
apt-get -y install postgresql
fi
if [ "$SQLITE3" = "yes" ]; then
echo "Installing sqlite3..."
apt-get -y install sqlite3
fi
exit 0
piuparts-0.56/custom-scripts/scripts/post_setup_squeeze-backports 0000777 0000000 0000000 00000000000 12145710343 031575 2post_distupgrade_squeeze-backports ustar piuparts-0.56/custom-scripts/scripts/pre_distupgrade_foreign_architecture_i386 0000775 0000000 0000000 00000001205 12253565512 025030 0 ustar #!/bin/sh
set -e
case "$PIUPARTS_DISTRIBUTION" in
squeeze*) ;;
*) exit 0 ;;
esac
case "$PIUPARTS_DISTRIBUTION_NEXT" in
lenny*|squeeze*) exit 0 ;;
esac
test "$(dpkg --print-architecture)" = "amd64" || exit 0
dpkg-query -s "ia32-libs" >/dev/null 2>&1 || exit 0
go=
case ${PIUPARTS_OBJECTS} in
*=None) go=yes ;;
esac
case ${PIUPARTS_OBJECTS%%=*} in
ia32-libs|ia32-libs-gtk) go=yes ;;
*wine*) go=yes ;;
education-thin-client-server|education-workstation) go=yes ;;
esac
test -n "$go" || exit 0
echo "Enabling foreign architecture i386 for $PIUPARTS_OBJECTS"
apt-get update
apt-get -y install apt dpkg
dpkg --add-architecture i386
piuparts-0.56/custom-scripts/scripts/pre_remove_40_find_obsolete_conffiles 0000775 0000000 0000000 00000001114 12144650665 024217 0 ustar #!/bin/sh
set -e
for pkg in ${PIUPARTS_OBJECTS%%=*}
do
dpkg-query -W -f='${Conffiles}\n' $pkg | \
grep ' obsolete$' | \
while read file md5expected obs
do
info="OBSOLETE CONFFILE $file REGISTERED BY $pkg"
query=$(dpkg-query -S $file)
owner=${query%: ${file}}
if [ "$owner" != "$pkg" ]; then
info="${info} OWNER CHANGED TO $owner"
fi
if [ ! -f "$file" ]; then
info="${info} (MISSING)"
else
md5=$(md5sum "$file" | awk '{ print $1 }')
if [ "$md5expected" != "$md5" ]; then
info="${info} (MODIFIED)"
fi
fi
echo "$info"
done
done
piuparts-0.56/custom-scripts/scripts/post_purge_exceptions 0000775 0000000 0000000 00000001365 12253565512 021262 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
#
# deal with packages depending on exceptions:
#
case ${PIUPARTS_OBJECTS%%=*} in
ltsp-client) log_debug
PIUPARTS_OBJECTS=ltsp-client-core
;;
esac
#
# deal with exceptions:
#
case ${PIUPARTS_OBJECTS%%=*} in
fai-nfsroot) log_debug
rm -f /.THIS_IS_THE_FAI_NFSROOT
;;
ltsp-client-core) log_debug
rm -f /etc/ltsp_chroot
;;
file-rc) log_debug
# removal wont work if sysv-rc isn't reinstalled
yes 'Yes, do as I say!' | apt-get -y --force-yes install sysv-rc
dpkg --purge file-rc
;;
amd64-libs|amd64-libs-dev)
# leaves a superfluous empty line after purge
log_debug
sed -i '3{/^$/d}' /etc/ld.so.conf
;;
esac
piuparts-0.56/custom-scripts/scripts-no-usr-share-doc/ 0000775 0000000 0000000 00000000000 12151074163 017755 5 ustar piuparts-0.56/custom-scripts/scripts-no-usr-share-doc/post_setup_disable_usr_share_doc 0000775 0000000 0000000 00000000531 12151074163 026472 0 ustar #!/bin/sh
set -e
case ${PIUPARTS_OBJECTS%%=*} in
dpkg)
# skip while creating the tarball
exit 0
;;
esac
# clear out /usr/share/doc
# but keep the directories (and symlinks) as well as the copyright files
cat << EOF > /etc/dpkg/dpkg.cfg.d/piuparts-path-exclude
path-exclude=/usr/share/doc/*/*
path-include=/usr/share/doc/*/copyright
EOF
piuparts-0.56/custom-scripts/scripts-debug-prerm/ 0000775 0000000 0000000 00000000000 12144650665 017111 5 ustar piuparts-0.56/custom-scripts/scripts-debug-prerm/pre_remove_prerm_set-x 0000775 0000000 0000000 00000000560 12144650665 023530 0 ustar #!/bin/sh
set -e
for target in ${PIUPARTS_OBJECTS%%=*}
do
pkg=${target}
prerm=/var/lib/dpkg/info/$pkg.prerm
if [ -f $prerm ]; then
if head -n 1 $prerm | grep -qE '/bin/(ba)?sh' ; then
echo "DEBUG PRERM REMOVE: enabling 'set -x' in $pkg.prerm"
sed -i '2 i set -x' $prerm
else
echo "Unsupported script type in $prerm:"
head -n 1 $prerm
fi
fi
done
piuparts-0.56/custom-scripts/scripts-debug-packagemanager/ 0000775 0000000 0000000 00000000000 12144650665 020712 5 ustar piuparts-0.56/custom-scripts/scripts-debug-packagemanager/pre_distupgrade_debug_packagemanager 0000775 0000000 0000000 00000000340 12144650665 030172 0 ustar #!/bin/sh
set -e
if [ ! -f /etc/apt/apt.conf.d/piuparts-debug-packagemanager ]
then
echo "Enabling Debug::pkgPackageManager"
echo 'Debug::pkgPackageManager "true";' >> /etc/apt/apt.conf.d/piuparts-debug-packagemanager
fi
piuparts-0.56/custom-scripts/scripts-debug-problemresolver/ 0000775 0000000 0000000 00000000000 12144650665 021206 5 ustar piuparts-0.56/custom-scripts/scripts-debug-problemresolver/pre_distupgrade_debug_problemresolver 0000775 0000000 0000000 00000000520 12144650665 030762 0 ustar #!/bin/sh
set -e
if [ ! -f /etc/apt/apt.conf.d/piuparts-debug-problemresolver ]
then
echo "Enabling Debug::pkgProblemResolver"
echo 'Debug::pkgProblemResolver "true";' >> /etc/apt/apt.conf.d/piuparts-debug-problemresolver
echo 'Debug::pkgProblemResolver::ShowScores "true";' >> /etc/apt/apt.conf.d/piuparts-debug-problemresolver
fi
piuparts-0.56/custom-scripts/scripts-debug-problemresolver/pre_install_debug_problemresolver 0000775 0000000 0000000 00000000575 12144650665 030127 0 ustar #!/bin/sh
set -e
test "$PIUPARTS_TEST" = "install" || exit 0
if [ ! -f /etc/apt/apt.conf.d/piuparts-debug-problemresolver ]
then
echo "Enabling Debug::pkgProblemResolver"
echo 'Debug::pkgProblemResolver "true";' >> /etc/apt/apt.conf.d/piuparts-debug-problemresolver
echo 'Debug::pkgProblemResolver::ShowScores "true";' >> /etc/apt/apt.conf.d/piuparts-debug-problemresolver
fi
piuparts-0.56/custom-scripts/scripts-leftovers/ 0000775 0000000 0000000 00000000000 12253565512 016706 5 ustar piuparts-0.56/custom-scripts/scripts-leftovers/pre_remove_preseed_cleanup 0000775 0000000 0000000 00000004063 12144650665 024223 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
dbconfig_enable_purge()
{
log_debug
echo "$1 $1${2+/$2}/dbconfig-remove boolean true" | debconf-set-selections
echo "$1 $1${2+/$2}/purge boolean true" | debconf-set-selections
}
#
# enable extended purge mode that is available in some packages
# but disabled by default because it might remove valuable user or
# application data
#
case ${PIUPARTS_OBJECTS%%=*} in
cvsd)
log_debug
echo "cvsd cvsd/remove_chroot boolean true" | debconf-set-selections
;;
docbookwiki)
log_debug
echo "docbookwiki docbookwiki/purge_books boolean true" | debconf-set-selections
;;
ifetch-tools)
log_debug
echo "ifetch-tools ifetch-tools/purge boolean true" | debconf-set-selections
;;
mlmmj) #668752
log_debug
echo "mlmmj mlmmj/remove-on-purge boolean true" | debconf-set-selections
;;
movabletype-opensource)
dbconfig_enable_purge movabletype-opensource
;;
nova-common)
dbconfig_enable_purge nova-common
;;
otrs2)
dbconfig_enable_purge otrs2
;;
pdns-backend-sqlite)
dbconfig_enable_purge pdns-backend-sqlite
;;
pdns-backend-sqlite3)
dbconfig_enable_purge pdns-backend-sqlite3
;;
redmine)
dbconfig_enable_purge redmine instances/default
;;
request-tracker3.8)
dbconfig_enable_purge request-tracker3.8
;;
request-tracker4)
dbconfig_enable_purge request-tracker4
;;
esac
if [ -d /var/lib/mysql ]; then
log_debug
echo "Enabling MySQL database purge."
echo "mysql-server-5.1 mysql-server-5.1/postrm_remove_databases boolean true" | debconf-set-selections
echo "mysql-server-5.5 mysql-server-5.5/postrm_remove_databases boolean true" | debconf-set-selections
fi
if [ -d /var/lib/ldap ]; then
log_debug
echo "Enabling LDAP database purge."
echo "slapd slapd/purge_database boolean true" | debconf-set-selections
fi
if [ -d /var/lib/cyrus ] || [ -d /var/spool/cyrus ] || [ -d /var/spool/sieve ]; then
log_debug
echo "Enabling Cyrus spool purge."
echo "cyrus-common cyrus-common/removespools boolean true" | debconf-set-selections
fi
exit 0
piuparts-0.56/custom-scripts/scripts-leftovers/post_remove_cleanup 0000775 0000000 0000000 00000000473 12144650665 022714 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
#
# deal with exceptions:
#
case ${PIUPARTS_OBJECTS%%=*} in
dovecot-core)
# #330519 - does not remove certificates on purge
log_debug
rm -fv /etc/dovecot/dovecot.pem /etc/dovecot/private/dovecot.pem
;;
esac
piuparts-0.56/custom-scripts/scripts-leftovers/post_purge_manual_cleanup 0000775 0000000 0000000 00000000714 12253565512 024071 0 ustar #!/bin/sh
set -e
log_debug()
{
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
remove_ssl_cert()
{
for c in /etc/ssl/certs/*.0 ; do
if [ -L "$c" ] && [ "$(readlink "$c")" = "$1" ]; then
rm -fv "$c"
fi
done
rm -fv /etc/ssl/certs/$1
rmdir --ignore-fail-on-non-empty /etc/ssl/certs
}
case ${PIUPARTS_OBJECTS%%=*} in
uw-imapd)
log_debug
remove_ssl_cert imapd.pem
;;
ipopd)
log_debug
remove_ssl_cert ipop3d.pem
;;
esac
piuparts-0.56/custom-scripts/scripts-leftovers/post_setup_fake-essential 0000775 0000000 0000000 00000001420 12253565512 024011 0 ustar #!/bin/sh
set -e
# If dbconfig-common was already purged, packages that have used
# dbconfig-common in postinst will leave configuration files in
# /etc/dbconfig-common
DBCONFIG="dbconfig-common"
# for purging configuration from /var/lib/systemd/
INITSYSTEMHELPERS="init-system-helpers"
case ${PIUPARTS_DISTRIBUTION} in
lenny*|squeeze*|wheezy*)
# package does not exist
INITSYSTEMHELPERS=""
;;
esac
# openssl may be used during purge to compute the hash for a
# certificate, otherwise files in /etc/ssl/certs can't be removed.
OPENSSL=""
case ${PIUPARTS_OBJECTS%%=*} in
dpkg)
# skip while creating the tarball
exit 0
;;
stone)
OPENSSL="openssl"
;;
esac
echo "*** Adding fake essential packages ***"
apt-get install -yf $DBCONFIG $INITSYSTEMHELPERS $OPENSSL
piuparts-0.56/custom-scripts/scripts-debug-purge/ 0000775 0000000 0000000 00000000000 12144650665 017106 5 ustar piuparts-0.56/custom-scripts/scripts-debug-purge/post_remove_postrm_set-x 0000775 0000000 0000000 00000000570 12144650665 024124 0 ustar #!/bin/sh
set -e
for target in ${PIUPARTS_OBJECTS%%=*}
do
pkg=${target}
postrm=/var/lib/dpkg/info/$pkg.postrm
if [ -f $postrm ]; then
if head -n 1 $postrm | grep -qE '/bin/(ba)?sh' ; then
echo "DEBUG POSTRM PURGE: enabling 'set -x' in $pkg.postrm"
sed -i '2 i set -x' $postrm
else
echo "Unsupported script type in $postrm:"
head -n 1 $postrm
fi
fi
done
piuparts-0.56/custom-scripts/scripts-squeeze/ 0000775 0000000 0000000 00000000000 12253565512 016356 5 ustar piuparts-0.56/custom-scripts/scripts-squeeze/post_distupgrade_squeeze-fake-essential 0000775 0000000 0000000 00000000505 12144650665 026321 0 ustar #!/bin/sh
set -e
test "$PIUPARTS_DISTRIBUTION" = "squeeze" || exit 0
case ${PIUPARTS_OBJECTS%%=*} in
kmplayer-plugin|kde-core|konqueror|konq-plugins|kwin-baghira|mozart-doc|smb4k|strigi-applet|texlive-full)
# work around #601961: apt: wrongly thinks install-info is essential
apt-get -y install install-info
;;
esac
piuparts-0.56/custom-scripts/scripts-squeeze/post_remove_exceptions_lenny 0000775 0000000 0000000 00000000570 12253565512 024316 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
case "$PIUPARTS_DISTRIBUTION" in
lenny) : ;;
*) exit 0 ;;
esac
case ${PIUPARTS_OBJECTS%%=*} in
postgis)
#WORKSAROUND #560409: postgis can't be purged if postgresql
# is not installed due to missing directory
log_debug
mkdir -p /usr/lib/postgresql/8.3/lib
;;
esac
piuparts-0.56/custom-scripts/scripts-squeeze/post_setup_squeeze-fake-essential 0000775 0000000 0000000 00000003205 12253565512 025143 0 ustar #!/bin/sh
set -e
# The following issues won't be fixed in squeeze:
# - unconditional use of deluser during postrm purge
# - unconditional use of ucf during postrm purge
# so add these packages to the "fake" essential set.
USED_DURING_PURGE="adduser ucf"
FAILS_TO_REMOVE=
case ${PIUPARTS_OBJECTS%%=*} in
dpkg)
# don't install fake essential packages while creating the tarball
exit 0
;;
adduser|ucf)
# allow testing of the fake essential packages
exit 0
;;
gosa-desktop|kde-core|kde-full|kde-plasma-desktop|kde-standard|kdebase-apps|konqueror|konq-plugins|mozart-doc|texlive-full)
# work around #601961: apt: wrongly thinks install-info is essential
if [ "$PIUPARTS_DISTRIBUTION" = "squeeze" ] || \
[ "$PIUPARTS_DISTRIBUTION" = "squeeze-proposed" ]; then
FAILS_TO_REMOVE="$FAILS_TO_REMOVE install-info"
fi
;;
docbookwiki)
USED_DURING_PURGE="$USED_DURING_PURGE mysql-client"
;;
phpbb3)
USED_DURING_PURGE="$USED_DURING_PURGE dbconfig-common"
;;
prelude-manager)
#WORKSAROUND #660455
USED_DURING_PURGE="$USED_DURING_PURGE dbconfig-common"
;;
drupal6|moodle|moodle-book|moodle-debian-edu-theme|scuttle)
if [ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
USED_DURING_PURGE="$USED_DURING_PURGE wwwconfig-common"
fi
;;
octave-audio|octave-symbolic|octave-vrml)
if [ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
USED_DURING_PURGE="$USED_DURING_PURGE octave3.0"
fi
;;
ttf-beteckna)
#502707
if [ "$PIUPARTS_DISTRIBUTION" = "lenny" ]; then
USED_DURING_PURGE="$USED_DURING_PURGE defoma"
fi
;;
esac
echo "*** Adding fake essential packages ***"
apt-get install -yf $USED_DURING_PURGE $FAILS_TO_REMOVE
piuparts-0.56/custom-scripts/scripts-wheezy/ 0000775 0000000 0000000 00000000000 12253565512 016210 5 ustar piuparts-0.56/custom-scripts/scripts-wheezy/pre_distupgrade_wheezy 0000775 0000000 0000000 00000004164 12253565512 022717 0 ustar #!/bin/sh
set -e
log_debug() {
echo "Debug: piuparts exception for package $PIUPARTS_OBJECTS"
}
# Work around bug where apt/squeeze prefers to 'keep back' some
# upgradable packages instead of removing some obsolete ones.
FORCEREMOVE=
if [ "$PIUPARTS_DISTRIBUTION_NEXT" = "wheezy" ] || \
[ "$PIUPARTS_DISTRIBUTION_NEXT" = "wheezy-proposed" ]
then
case ${PIUPARTS_OBJECTS%%=*} in
gnustep|\
gnustep-games|\
gnustep-back0.18-cairo|\
gnustep-back0.18-art|\
gnustep-back-dbg|\
libgnustep-base1.20-dbg|\
addressmanager.app|\
biococoa.app|\
easydiff.app|\
gnumail.app|\
gnumail.app-dbg|\
mines.app|\
projectmanager.app|\
INVALID)
FORCEREMOVE="libobjc2"
;;
libahven17.0|\
libalog0.3-full|\
libalog1-full-dev|\
libalog-full-dbg|\
libapq3.0|\
libapq-postgresql3.0|\
libaws2.7|\
libaws2.7-dev|\
libaws-dbg|\
libflorist2009|\
libflorist-dbg|\
libgmpada1|\
libgmpada1-dbg|\
libgnomeada2.14.2|\
libgnomeada2.14.2-dbg|\
libgnomeada2.14.2-dev|\
libgtkada2.14.2|\
libgtkada2.14.2-dbg|\
libgtkada2.14.2-dev|\
libgtkada-gl2.14.2|\
libgtkada-glade2.14.2|\
libgtkada2-bin|\
liblog4ada0|\
liblog4ada-dbg|\
libnarval1.10.1|\
libnarval1-dev|\
libnarval-dbg|\
libpcscada0.6|\
libplplot-ada|\
libplplot-dev|\
libpolyorb2|\
libtemplates-parser11.5|\
libtexttools4|\
libtexttools4-dbg|\
libtexttools-doc|\
libxmlada3.2|\
libxmlezout0|\
libxmlezout-dbg|\
cl-plplot|\
gprbuild|\
narval-doc|\
narval-generic-actors|\
narval-servers|\
narval-tests-actors|\
narval-utils|\
polyorb-servers|\
INVALID)
FORCEREMOVE="gnat-4.4-base"
;;
dolfin-bin|\
dolfin-dev|\
dolfin-doc|\
petsc-dev|\
python-dolfin|\
INVALID)
# only affects i386
FORCEREMOVE="libjpeg62-dev"
;;
science-physics)
# #706111
FORCEREMOVE="tessa"
;;
firestarter|\
gnome-netstatus-applet|\
gnome-utils|\
INVALID)
FORCEREMOVE="gconf2"
;;
esac
fi
if [ -n "$FORCEREMOVE" ];
then
if dpkg-query -s $FORCEREMOVE >/dev/null 2>&1 ; then
log_debug
echo "Forcibly removing $FORCEREMOVE for smoother upgrade"
dpkg -r --force-depends $FORCEREMOVE
fi
fi
piuparts-0.56/custom-scripts/scripts-wheezy/post_setup_wheezy-fake-essential 0000775 0000000 0000000 00000001046 12144653075 024631 0 ustar #!/bin/sh
set -e
# The following issues won't be fixed in wheezy:
# - unconditional use of deluser during postrm purge
# - unconditional use of ucf during postrm purge
# so add these packages to the "fake" essential set.
USED_DURING_PURGE="adduser ucf"
case ${PIUPARTS_OBJECTS%%=*} in
dpkg)
# don't install fake essential packages while creating the tarball
exit 0
;;
adduser|ucf)
# allow testing of the fake essential packages
exit 0
;;
esac
echo "*** Adding fake essential packages ***"
apt-get install -yf $USED_DURING_PURGE
piuparts-0.56/Makefile 0000664 0000000 0000000 00000014412 12155061511 011642 0 ustar prefix = /usr/local
sbindir = $(prefix)/sbin
sharedir = $(prefix)/share
mandir = $(sharedir)/man
man1dir = $(mandir)/man1
man8dir = $(mandir)/man8
libdir = $(prefix)/lib
docdir = $(prefix)/share/doc/piuparts/
site27 = $(libdir)/python2.7/dist-packages
htdocsdir = $(sharedir)/piuparts/htdocs
etcdir = $(prefix)/etc
distribution=${shell dpkg-parsechangelog | sed -n 's/^Distribution: *//p'}
ifeq ($(distribution),UNRELEASED)
version := ${shell echo "`dpkg-parsechangelog | sed -n 's/^Version: *//p'`~`date +%Y%m%d%H%M`~`git describe --dirty`"}
else
version := ${shell dpkg-parsechangelog | sed -n 's/^Version: *//p'}
endif
# generate several scripts, conffiles, ... from templates (*.in, *.py)
# by substituting placeholders
SCRIPTS_TEMPLATES = $(wildcard *.in master-bin/*.in slave-bin/*.in conf/*.in)
SCRIPTS_PYTHON_BINARY = $(wildcard *.py master-bin/*.py slave-bin/*.py)
SCRIPTS_GENERATED = $(SCRIPTS_TEMPLATES:.in=) $(SCRIPTS_PYTHON_BINARY:.py=)
DOCS_GENERATED = piuparts.1 piuparts.1.html piuparts_slave_run.8 piuparts_slave_join.8 README_1st.html README_server.html
define placeholder_substitution
sed -r \
-e 's/__PIUPARTS_VERSION__/$(version)/g' \
-e 's%@sharedir@%$(sharedir)%g' \
$< > $@
endef
%: %.in Makefile
$(placeholder_substitution)
%: %.py Makefile
$(placeholder_substitution)
all: build
python_scripts = $(wildcard *.py piupartslib/*.py master-bin/*.py slave-bin/*.py)
python-syntax-check:
@set -e -x; $(foreach py,$(python_scripts),python -m py_compile $(py);)
$(RM) $(python_scripts:=c)
build: build-stamp
build-stamp: $(SCRIPTS_GENERATED) $(DOCS_GENERATED) Makefile
$(MAKE) python-syntax-check
touch $@
build-doc: $(DOCS_GENERATED)
README_1st.html: README_1st.txt
a2x --copy -a toc -a toclevels=3 -f xhtml -r /etc/asciidoc/ README_1st.txt
README_server.html: README_server.txt
a2x --copy -a toc -a toclevels=3 -f xhtml -r /etc/asciidoc/ README_server.txt
piuparts.1: piuparts.1.txt
a2x -f manpage piuparts.1.txt
piuparts_slave_run.8: piuparts_slave_run.8.txt
a2x -f manpage piuparts_slave_run.8.txt
piuparts_slave_join.8: piuparts_slave_join.8.txt
a2x -f manpage piuparts_slave_join.8.txt
piuparts.1.html: piuparts.1.txt
a2x --copy -f xhtml piuparts.1.txt
install-doc: build-stamp
install -d $(DESTDIR)$(docdir)/
install -m 0644 README_1st.txt README_1st.html README_server.txt README_server.html docbook-xsl.css $(DESTDIR)$(docdir)/
install -d $(DESTDIR)$(man1dir)
install -m 0644 piuparts.1 $(DESTDIR)$(man1dir)/
install -d $(DESTDIR)$(man8dir)
install -m 0644 piuparts_slave_run.8 piuparts_slave_join.8 $(DESTDIR)$(man8dir)/
gzip -9f $(DESTDIR)$(man1dir)/piuparts.1
gzip -9f $(DESTDIR)$(man8dir)/piuparts_slave_run.8
gzip -9f $(DESTDIR)$(man8dir)/piuparts_slave_join.8
install -m 0644 piuparts.1.html $(DESTDIR)$(docdir)/
install-conf: build-stamp
install -d $(DESTDIR)$(etcdir)/piuparts
install -m 0644 conf/piuparts.conf.sample $(DESTDIR)$(etcdir)/piuparts/piuparts.conf
install -m 0644 conf/distros.conf $(DESTDIR)$(etcdir)/piuparts/
install -d $(DESTDIR)$(etcdir)/cron.d
install -m 0644 conf/crontab-master $(DESTDIR)$(etcdir)/cron.d/piuparts-master
install -m 0644 conf/crontab-slave $(DESTDIR)$(etcdir)/cron.d/piuparts-slave
# disable shipped crontabs
sed -i -r '/^[^#]+/s/^/#/' $(DESTDIR)$(etcdir)/cron.d/piuparts-*
install -d $(DESTDIR)$(etcdir)/sudoers.d
install -m 440 conf/piuparts.sudoers $(DESTDIR)$(etcdir)/sudoers.d/piuparts
# disable shipped sudoers
sed -i -r '/^[^#]+/s/^/#/' $(DESTDIR)$(etcdir)/sudoers.d/piuparts
install -d $(DESTDIR)$(etcdir)/apache2/conf.d
install -m 0644 conf/piuparts.apache $(DESTDIR)$(etcdir)/apache2/conf.d/
install-conf-4-running-from-git: build-stamp
install -d $(DESTDIR)$(etcdir)/piuparts
install -m 0644 conf/crontab-master $(DESTDIR)$(etcdir)/piuparts/
install -m 0644 conf/crontab-slave $(DESTDIR)$(etcdir)/piuparts/
install -m 0644 conf/distros.conf $(DESTDIR)$(etcdir)/piuparts/
install -m 0644 instances/piuparts.conf.* $(DESTDIR)$(etcdir)/piuparts/
install -d $(DESTDIR)$(sharedir)/piuparts/slave
install -m 0755 update-piuparts-slave-setup $(DESTDIR)$(sharedir)/piuparts/slave/
install -d $(DESTDIR)$(sharedir)/piuparts/master
install -m 0755 update-piuparts-master-setup $(DESTDIR)$(sharedir)/piuparts/master/
install: build-stamp
install -d $(DESTDIR)$(sbindir)
install -m 0755 piuparts $(DESTDIR)$(sbindir)/
install -d $(DESTDIR)$(sharedir)/piuparts
install -m 0755 piuparts-slave piuparts-master piuparts-master-backend piuparts-report piuparts-analyze $(DESTDIR)$(sharedir)/piuparts/
install -d $(DESTDIR)$(site27)/piupartslib
install -m 0644 piupartslib/*.py $(DESTDIR)$(site27)/piupartslib/
install -d $(DESTDIR)$(sharedir)/piuparts/lib
install -m 0644 lib/*.sh $(DESTDIR)$(sharedir)/piuparts/lib/
# do not install the templates (*.in, *.py)
install -d $(DESTDIR)$(sharedir)/piuparts/master
install -m 0755 $(filter-out %.in %.py,$(wildcard master-bin/*)) $(DESTDIR)$(sharedir)/piuparts/master/
install -d $(DESTDIR)$(sharedir)/piuparts/known_problems
install -m 0644 known_problems/*.conf $(DESTDIR)$(sharedir)/piuparts/known_problems/
# do not install the templates (*.in, *.py)
install -d $(DESTDIR)$(sharedir)/piuparts/slave
install -m 0755 $(filter-out %.in %.py,$(wildcard slave-bin/*)) $(DESTDIR)$(sharedir)/piuparts/slave/
install -d $(DESTDIR)$(htdocsdir)
install -m 0644 htdocs/*.* $(DESTDIR)$(htdocsdir)/
install -d $(DESTDIR)$(htdocsdir)/images
install -m 0644 htdocs/images/*.* $(DESTDIR)$(htdocsdir)/images/
install -d $(DESTDIR)$(htdocsdir)/templates/mail
install -m 0644 bug-templates/*.mail $(DESTDIR)$(htdocsdir)/templates/mail/
install -d $(DESTDIR)$(etcdir)/piuparts
@set -e -x ; \
for d in $$(ls custom-scripts) ; do \
install -d $(DESTDIR)$(etcdir)/piuparts/$$d ; \
install -m 0755 custom-scripts/$$d/* $(DESTDIR)$(etcdir)/piuparts/$$d/ ; done
#install -d $(DESTDIR)$(etcdir)/piuparts/known_problems
#install -m 0644 known_problems/*.conf $(DESTDIR)$(etcdir)/piuparts/known_problems/
check:
python piuparts.py unittest
python unittests.py
clean:
rm -f build-stamp
rm -f $(DOCS_GENERATED)
rm -f piuparts.1.xml README.xml README_server.xml docbook-xsl.css piuparts.html
rm -f *.pyc piupartslib/*.pyc master-bin/*.pyc slave-bin/*.pyc
rm -f $(SCRIPTS_GENERATED)
# for maintainer convenience only
tg-deps:
tg summary --graphviz | dot -T png -o deps.png
xli deps.png &
piuparts-0.56/piuparts-master.in 0000775 0000000 0000000 00000002437 12151074163 013704 0 ustar #!/bin/sh
set -e
# Copyright © 2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# wrapper for running piuparts-master-backend without needing to know
# anything about the master setup and paths
#
. @sharedir@/piuparts/lib/read_config.sh
get_config_value MASTER global master-directory
get_config_value PYTHONPATH global PYTHONPATH ''
get_config_value LOGFILE global log-file master-error.log
export PYTHONPATH
LOGFILE="$LOGFILE.$$"
# put logfile in a deterministic location
cd "$MASTER"
trap "test -s ${LOGFILE} || rm -f ${LOGFILE}" EXIT
timeout 15m @sharedir@/piuparts/piuparts-master-backend 2> "$LOGFILE"
piuparts-0.56/debian/ 0000775 0000000 0000000 00000000000 12323207374 011430 5 ustar piuparts-0.56/debian/changelog 0000664 0000000 0000000 00000263135 12323207374 013314 0 ustar piuparts (0.56ubuntu1) trusty; urgency=medium
* Change default Ubuntu distribution to Trusty
-- Zygmunt Krynicki Tue, 15 Apr 2014 12:46:15 +0200
piuparts (0.56) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Add to ignored_files:
+ /etc/apt/apt.conf.d/01autoremove-kernels
- Use temporary files while creating basetgz and rename afterwards.
- Remove the package to be tested separately before removing its Depends
and Recommends.
- --allow-database now also enables firebird2.5-super.
* piuparts.conf:
- New per-section settings:
+ components
* piupartslib/__init__.py:
- open_packages_url(): Try Packages.gz if Packages.bz2 is not available.
(Closes: #711157)
- Add new class DecompressedStream.
- open_packages_url(): Use DecompressedStream to transparently decompress
a Packages.{bz2,gz} file while downloading and reading the content
without requiring to store the complete file (compressed and/or
decompressed) in memory.
* piupartslib/packagesdb.py:
- PackagesFile: Allow restricting to a set of package names.
* piuparts-slave.py:
- Use locking to prevent concurrent basetgz creation. (Closes: #708287)
- Use the "components" setting to restrict the archive areas being used
in the generated sources.list.
- Reduce memory footprint by discarding unneeded Packages file parts.
* piuparts-report.py:
- Report URLs of all Packages files used for a section.
- Avoid reporting duplicate dependencies after stripping versioning.
- Fix some invalid HTML constructs.
- Sort maintainer summary by state: failing packages on top.
* Improve exception handling.
* known_problems: Upgrade adequate issue 'missing-copyright-file' from
boring to normal and make pre_remove_50_find_missing_copyright no longer
generate fatal errors.
* Add support for new adequate tags missing-symbol-version-information and
symbol-size-mismatch.
* reschedule_oldest_logs: Report number of logfiles in recycle/.
* Add more piuparts exceptions to work around various failures in multiple
packages. Add a known_problem report for packages using exceptions.
* bug-templates: Recommend to use the new dpkg-maintscript-helper commands
dir_to_symlink/symlink_to_dir that were added in dpkg 1.17.2.
(Closes: #720185)
[ Holger Levsen ]
* piuparts-report.py: use descriptive link targets for (in)adequate issues
instead of mere tag names, thanks to Jakub Wilk for the wordings.
* Bump standards version to 3.9.5, no changes needed.
* Bump compat level to 9, thus build-depend on debhelper > 9.20120909~.
-- Holger Levsen Mon, 16 Dec 2013 12:59:14 +0100
piuparts (0.55) unstable; urgency=low
[ Ivo De Decker ]
* piuparts-report.py: Add sources.yaml and section.yaml exports.
[ Holger Levsen ]
* Add file CONTRIBUTING, thanks to Vincent McIntyre for it! (Closes: #721348)
* piuparts-report: Add "contact us" link.
* piuparts.py: Explain at the very beginning of the generated log that
contact information is part of the FAQ.
[ Andreas Beckmann ]
* Add bug templates for wheezy.
* scripts-leftovers: Improve cleanup in /var/lib/systemd with fake-essential
init-system-helpers. (Closes: #719869)
-- Holger Levsen Thu, 03 Oct 2013 15:31:38 +0200
piuparts (0.54) unstable; urgency=low
[ Holger Levsen ]
* known_problems: fix some adequate patterns.
* piuparts-master: recommend apache2 | httpd.
* README_server.txt: be more verbose about configuring sudo.
(Closes: #711155)
* piuparts.py and piuparts-reports.py: add "incompatible-licenses" (detected
since adequate 0.7) and "ldd" to inadequate_tags and known_problems.
[ David Steele ]
* piupartslib/packagesdb.py
- The dependency calculation for rdeps is reworked to match that used
elsewhere, to allow package reserve() to drive waiting-for-dep to
drive to 0 more quickly. (Closes: #705977)
- Reworked rdep metrics (block_count, rrdep_count, waiting_count) to
calculate on-demand. Access methods are moved from Package to
PackagesDB.
- Add a metric for the worst-case reverse-dependency chain length.
- Weight the reserved packages returned by this additional factor.
[ Andreas Beckmann ]
* piuparts.py: add to ignored_files: /etc/hosts
-- Holger Levsen Tue, 30 Jul 2013 11:05:32 +0200
piuparts (0.53) unstable; urgency=low
[ Andreas Beckmann ]
* *.py: Reindent and reformat, slowly moving towards PEP 8 style.
* piuparts.py:
- Do not run adequate checks on possibly disappearing packages.
- Add --allow-database option that enables starting MySQL and PostgreSQL
database servers in the chroot. (Closes: #662633)
- Relax the check for running processes if --allow-database is given.
* distros.conf:
- Add "candidates" key to merge multiple Packages files into one virtual
distro.
- Define wheezy-proposed and squeeze-proposed.
* piupartslib/conf.py:
- Add get_{packages,sources}_urls() that resolve "candidates".
* piupartslib/packagesdb.py:
- Add load_packages_urls() methods.
* piuparts-{master-backend,slave,report}.py:
- Switch to load_*_urls(get_*_urls()) for loading Packages/Sources files,
adding support for arbitrarily composed virtual distros.
* Makefile:
- Stop installing python modules in $libdir/python2.6/dist-packages.
* debian/control:
- Bump Build-Depends: python (>= 2.7).
- piuparts-slave: Add Depends: sudo. (Closes: #710350)
* New known problem: Packages with missing or incomplete .md5sums files.
* piuparts-master.deb: Ship some piuparts.conf examples.
* detect_piuparts_issues: Detect problems starting MySQL or PostgreSQL.
* pre_install_database-server: New script to install a database server for
packages that require one during postinst but only recommend or suggest
(or maybe not even this) the server package to allow for remote db
servers.
* pre_distupgrade_zz_database-server: New script to upgrade (running)
database servers before all other packages. This ensures the database
server is running again before the dependent packages (that may declare
'insufficient' dependencies on a 'local' database server) are upgraded.
[ David Steele ]
* Update python Requires to 2.7 (for argparse)
* Edit piuparts_slave_* man pages for more information and format.
* Specify the manpage doctype for AsciiDoc man sources.
[ Holger Levsen ]
* piuparts.1.txt, README.txt and README_pejacevic.txt: proof-read and
cleaned up.
* Move master-slave related documentation from README.txt to
README_server.txt and leave a pointer in the former.
* Rename README.txt to README_1st.txt.
* piuparts-master.deb:
- Ship README_server.txt.
- Recommend devscripts and libsoap-lite-perl (for gather_bts_stats).
* piuparts-report.py: Add a link to README_server.html.
* master-bin/gather_bts_stats: write bts_stats.txt to master-directory.
* conf/crontab-master: run gather_bts_stats daily.
* master-bin/prepare_backup: backup bts_stats.txt and do not backup
sources.txt.
* piuparts.py: categorize adequate results using inadequate_tags,
boring_tags, ignored_tags and unknown. Show adequate exit code if
non-zero.
* Add known_problems/*inadequate*.conf for all adequate tags we care about.
* htdocs/index.tpl: Improve description about piuparts.debian.org.
(Closes: #700848)
[ Andrew Starr-Bochicchio ]
* piuparts.py:
- Move default keyring to distro defaults. (Closes: #710346)
- When distro defaults are explicitly specified, use the distro default
mirrors (Closes: #710347).
-- Holger Levsen Sun, 09 Jun 2013 12:57:52 +0200
piuparts (0.52) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Really disable PDiffs. (Closes: #708241)
* piuparts-master.in:
- Use timeout to prevent master from getting stuck. (Closes: #701618)
* piuparts-slave.py:
- Report upgrade tests involving downgrades as untestable.
- Always pass --arch to piuparts.
* piuparts-analyze.py:
- Consider archived bugs, too. (Closes: #669966)
* piuparts-report.py:
- Switch from python-rpy to python-rpy2. (Closes: #705975)
- rpy2: Use a noninteractive grDevice.
* Makefile:
- Switch default target from 'install' to 'build'.
* scripts-no-usr-share-doc/post_setup_disable_usr_share_doc: New custom
script to instruct dpkg to skip installing most files in /usr/share/doc.
This catches maintainer scripts relying on /usr/share/doc content.
* New bug template: postinst_uses_usr_share_doc.
* piuparts-master.deb:
- Switch Depends: python-rpy to Depends: python-rpy2.
[ Holger Levsen ]
* piuparts.py:
- Run adequate (if installed) after installation test. New options
--no-adequate and --fail-if-inadequate. (Closes: #703902)
- Workaround #709372 and ignore broken symlinks found by adequate.
- Do not print a "Guessed: debian" line on every run. It's the default
anyway. (Closes: #703058)
* piuparts-report:
- Add templates for known_problems/inadequate_issue and inadequate_error.
- Add link to the piuparts source at the top of the navigation menu on the
left.
- Add links to Debian QA group and jenkins.debian.net at the bottom of
that navigation menu.
* debian/control: Add adequate to piuparts' recommends.
* piuparts-slave.postrm: Also remove /var/cache/piuparts/tmp on purge.
* conf/crontab-master: Run generate_daily_report twice a day.
* piuparts-slave.deb: Add piuparts_slave_run.8 and piuparts_slave_join.8
manpages.
-- Holger Levsen Mon, 27 May 2013 18:10:43 +0200
piuparts (0.51) unstable; urgency=low
[ Andreas Beckmann ]
* Mark all python code as utf-8 encoded.
* piuparts.py:
- In install-purge tests support installing extra packages before the
package to be tested via --extra-old-packages.
- Add --arch option to run the test for the requested instead of
the default architecture.
- Disable using PDiffs in order to save CPU instead of bandwidth to a
local mirror/proxy. (Closes: #708241)
* piuparts.conf:
- New global settings:
+ slave-count
- Use installation-dependent defaults for known-problem-directory.
* distros.conf:
- Update backports setup after integration into the regular archive.
- Enable stable security updates when installing stable backports.
- Set testing = jessie and add jessie-proposed-updates.
* piupartslib/conf.py:
- Raise MissingSection if the requested section is not in piuparts.conf.
- Improve handling of virtual distros.
* piupartslib/packagesdb.py:
- get_package(): Add flag resolve_virtual, disabled by default.
* Rename piuparts-master.py to piuparts-master-backend.py
* piuparts-master: New.
- Wrapper shell script to simplify calling the master from the slave.
- Get the 'master-directory' from piuparts.conf and chdir there.
- Get the (global) 'log-file' from piuparts.conf and handle the error
logging. No longer uses the section specific logfile name for errors.
* Allow the master-command to be set in ~piupartsm/.ssh/authorized_keys to
restrict ssh keys to only run piuparts-master.
* piuparts-master-backend.py:
- Raise CommandSyntaxError on unknown commands.
- Add "section" command:
+ Make section selection part of the master-slave protocol.
+ Allow switching sections without reconnecting.
+ No longer accept section argument on the command line.
+ Switch the logfile after switching sections.
* piuparts-slave.py:
- Fix parsing 'idle' return value from master.
- Let the piuparts-master (wrapper script) handle chdir and error logging.
- Skip sections that don't exist in piuparts.conf and continue to run.
- Pass the section to piuparts-master via the new "section" command
instead of using a command line argument.
- Keep the connection to the master open while switching sections. This
reduces the number of ssh connections attempted while checking idle
sections. Close the connection before processing packages, before going
to sleep, and after communication errors.
- Support empty master-command if the command is set in master's
.ssh/authorized_keys file.
- Change default basetgz name to {DISTRO}_{ARCH}.tar.gz.
* piuparts-report.py:
- Skip sections that don't exist in piuparts.conf.
- "Missing" binary packages don't influence the overall test result of a
source package. These are either udebs or for other architectures.
(Closes: #699160)
- Report the correct version for binary packages.
* archive_old_logs:
- Compress the archived logs.
* detect_well_known_errors:
- Skip sections that don't exist in piuparts.conf.
* piuparts-master.deb:
- Add Depends: xz-utils.
* piuparts-slave.deb:
- Restrict the ssh key added to master's authorized_keys to only run
piuparts-master.
* slave_run: Support starting multiple concurrent slaves. (Closes: #678725)
* Update and add new exceptions for buggy packages.
* scripts-wheezy/post_setup_wheezy-fake-essential: New custom script to
suppress some purge failures in wheezy.
* post_{setup,distupgrade}_squeeze-backports: New custom scripts to improve
support for squeeze-backports.
* New known problem: Database setup failure (issue).
[ David Steele ]
* detect_well_known_errors:
- Replace the bash script with an equivalent Python script.
- Sort known error and issue packages by reverse dependency count,
separating 'fail' from everything else.
(Closes: #698526)
- Add a PTS link to issue and error summary entries.
- Replace the known_problem COMMAND with PATTERN and EXCLUDE_PATTERN,
and replace grep shell calls with python re modules calls, for a
10x speedup.
- Validate field names, and only use valid known problem conf files.
- Minor HTML syntax fix.
- Minor template integration.
[ Holger Levsen ]
* piuparts.py:
- Add to ignored_files:
+ /var/log/fontconfig.log (Closes: #703810) - Thanks to Hideki Yamane.
- switch default Ubuntu distribution to "saucy".
* Bump Standards-Version to 3.9.4.
* Use /srv/piuparts.debian.org/ on piatti.debian.org as per DSA policy.
* Provide instance configuration for pejacevic.d.o and piu-slave-bm-a.d.o.
* Move README_piatti.txt and README_pejacevic.txt and update accordingly.
* Provide two new helper scripts for running from git:
update-piuparts-master-setup and update-piuparts-slave-setup, drop
update-piuparts-setup.
* Drop instances/forward.* files.
* slave-bin/slave_run:
- Run slave_cleanup before starting the slave.
* slave-bin/detect_slave_problems: remove check limiting this script to be
run on piatti only.
* master-bin/prepare_backup: also backup sources.txt.
-- Holger Levsen Wed, 15 May 2013 11:31:31 +0200
piuparts (0.50) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Add to ignored_files:
+ /var/lib/dpkg/arch
+ /usr/lib/python2.[67]/dist-packages/python-support.pth
broken symlinks caused by python-support (see #635493 and #385775,
thanks David Steele)
- Call dpkg-deb with --nocheck to allow bad version numbers.
- Pass unqualified package names to apt-cache show for lenny support.
- Support plain URLs or local paths as --testdebs-repo argument and
prepend/append "deb", "file://", and "./" as needed.
- Improve the "dirname part contains a symlink" test and report
overwritten files owned by other packages as failures.
- Add --proxy=URL option.
- piuparts.deb: Add Depends: piuparts-common for using distros.conf and
piupartslib.
- Use distros.conf to generate mirror configuration. This enables more
complex scenarios involving partial distributions and dist-upgrades.
- Use apt-get -t if set in distros.conf.
* piuparts.conf:
- New global settings:
+ proxy
+ slave-directory
+ known-problem-directory (for known_problem definitions)
+ backup-directory (for the master script prepare_backup)
+ PYTHONPATH (to find the python modules in non-default path setups)
+ slave-load-max
- New per-section settings:
+ depends-sections
+ basetgz-directory
- Obsolete settings: upgrade-test-chroot-tgz.
- Setting tmpdir is now mandatory for master/slave operation.
* distros.conf:
- New configuration file to describe full distributions (e.g. sid,
squeeze) and partial distributions (e.g. squeeze/security,
squeeze-backports, experimental) along with their dependencies (e.g.
base distribution) and non-default mirrors. (Closes: #699028)
- Allows setting a target-release (for apt-get -t ...). (Closes: #687995)
- Shipped in package piuparts-common.
* piupartslib/conf.py:
- Add methods: get_distros(), get_start_distro().
- Add class DistroConfig for reading distros.conf.
* piupartslib/packagesdb.py:
- Add Package.waiting_count() and friends, populated in calc_rrdep_count.
- Optimize reserve() and skip unavailable candidates.
- Add method: set_dependency_databases().
- Lookup missing packages, states, ... in the dependency_databases.
* piuparts-master.py:
- Sort packages to be tested by importance, i.e. descending waiting_count.
- Use piuparts.conf global proxy setting as http_proxy.
- Use distros.conf to compute URLs.
- Load depends-sections for package lookup in partial distros.
* piuparts-slave.py:
- Tarball recreation can be disabled by setting max-tgz-age to 0.
- Relax package existence checks to allow distupgrading to backports.
- Fix SIGHUP handling while recycling.
- Flush and unreserve all sections on interrupt.
- Add support for running piuparts under /usr/bin/timeout.
- Check system load frequently and enter sleep mode if threshold
(piuparts.conf global setting slave-load-max) is exceeded. Operation
will be resumed after load drops below 'slave-load-max - 1.0'. Disabled
by default.
- Use piuparts.conf global proxy setting as http_proxy and pass it to
piuparts via --proxy.
- Use distros.conf to compute URLs.
- Use the upgrade-test-distros setting to select between install/purge
test (if unset) and dist-upgrade test (if there are at least 2 distros).
Drop support for running both tests for a section.
* piuparts-analyze.py:
- Add support for magic "$DISTRO/None" versions.
* piuparts-report.py:
- Call r.dev_off() after generating a plot. (Closes: #657799)
- Use piuparts.conf global proxy setting as http_proxy.
- Use distros.conf to compute URLs.
- Generate/prepend/append a default description.
- Load depends-sections for package lookup in partial distros.
* Makefile:
- Add DESTDIR support.
- Overhaul. Add dependencies between build and install targets.
- Add support for placeholder substitution in scripts and config.
- Perform syntax check on *.py before installation.
* debian/rules: Set 'prefix' and 'DESTDIR' for build and install.
* lib/read_config.sh: Factored out the piuparts.conf reading function that
is used by all master/slave shell scripts. Shipped in piuparts.common.
- Add support for RFC822 line continuation (wrap + indent) and comments.
* master-bin/slave-bin: Switch to sourcing read_config.sh.
* conf/crontab*: Substitute @sharedir@ at build time.
* known_problems: Install to @sharedir@/piuparts/known_problems/.
* Replace all remaining references to $HOME/bin and ~/bin with @sharedir@.
* prepare_backup:
- Get paths from piuparts.conf.
- Prepare a copy of submissions.txt, too.
* master-bin/slave-bin: Use PYTHONPATH from piuparts.conf.
* piuparts-slave.deb:
- Use /var/cache/piuparts/tmp as tmpdir.
- Use /var/cache/piuparts/basetgz as basetgz-directory.
- Populate ~piupartss/.ssh/known_hosts with localhost's hostkey.
- Restrict the ssh key added to master's authorized_keys to only run
piuparts-master.
* slave_{run,join}: Rename the screen session to "piuparts_slave_screen".
* slave_cleanup: Use pgrep to find running piuparts-slave processes.
* slave_{run,join}: Use screen -ls to find running screen sessions.
* slave_join: Use 'script /dev/null' hack if needed to get a writable tty.
This avoids running the piuparts-slave screen session as root.
* slave_run: Get paths etc. from piuparts.conf.
* slave_run: Ensure the tmpdir exists, this could be on tmpfs.
* detect_leftover_processes: Move statefile to slave-directory.
* detect_stale_mounts: Merge into detect_tmp_cruft.
* detect_tmp_cruft: Move to slave-bin and run from slave crontab.
* detect_slave_problems:
- Move to slave-bin and run from slave crontab.
- Use idle-sleep setting from piuparts.conf.
- Move statefile to slave-directory.
* master: Get rid of the monitor-slave directory.
* pre_install_exceptions: Handle bootcd-ia64 installation in squeeze.
* post_distupgrade_exceptions: Handle libdb4.8 removal in wheezy.
* pre_remove_50_find_bad_permissions: Add some exceptions.
* pre_remove_50_find_missing_copyright: Add some exceptions.
* post_{setup,distupgrade}_experimental: New custom scripts to improve
support for experimental.
* Add several exceptions for lenny and lenny2squeeze tests.
* New script to enable Debug::pkgPackageManager during upgrades:
scripts-debug-packagemanager/pre_distupgrade_debug_packagemanager
* New known problem: Packages that need to be rebuild or updated for newer
tools, e.g. due to obsolete install-info or tex-common usage.
* New bug template: unhandled_symlink_to_directory_conversion.
* New bug template: installs_over_existing_symlink.
[ Holger Levsen ]
* Mention http://piuparts.debian.org in README.txt prominently.
* Mention github clone in htdocs/index.tpl.
[ David Steele ]
* Replace the Tango icon symlinks with copies.
* Make the Tango weather icons required for master.
-- Holger Levsen Fri, 15 Mar 2013 15:36:12 -0700
piuparts (0.49) unstable; urgency=low
[ Andreas Beckmann ]
* Add support for dist-upgrade tests of "disappearing" packages:
- master/report: Get candidate packages from distro in the "distro"
setting in piuparts.conf and (target) versions to test from last distro
entry in "upgrade-test-distros". In case a package no longer exists, use
a pseudo-version "None". (Closes: #656438)
- slave: Support pseudo-version "None".
- piuparts: Do not try to install unavailable packages after dist-upgrade.
* piuparts.py:
- Run custom scripts only once if dependencies are installed via a
metapackage.
- Rename --no-debsums option to --warn-on-debsums-errors.
- Add --extra-repo= option for adding extra
package sources, e.g. backports, security or local repositories.
The argument must be a valid line (including whitespace) that is added
to sources.list verbatim. The option may be repeated to add more lines.
- Qualify to the package names extracted from .debs with the version.
- Add --testdebs-repo= option. Provide the package(s)
to be tested (and additional dependencies that are not yet in the
archive) in a repository to enable dependency resolution with apt-get.
Use for complex install and upgrade scenarios. (Closes: #676694)
- Add --distupgrade-to-testdebs option. During distupgrade tests override
packages from the target distro with the packages from the testdebs
repository. This allows doing distupgrade tests before uploading.
- Enable upgrade tests if both --apt and --testdebs-repo are given.
* piupartslib/conf.py:
- Add get_final_distro() method.
- Add optional distro argument to get_mirror() method.
* piupartslib/packagesdb.py:
- Speed up LogDB by fetching existing files with os.listdir().
* piuparts-slave.py:
- Disable X forwarding while connecting to master.
- Move checks for package existence to test_package().
* piuparts-analyze.py:
- Sort bugs and try the newest bug first.
* piuparts-report.py:
- Fix URLs to piatti's config.
- Hardlink the logfiles to the htdocs tree (with copy as fallback).
* post_setup_squeeze-fake-essential: Restrict fake-essential install-info to
the 4 packages that actually need this.
* New custom script: pre_remove_40_find_obsolete_conffiles, report obsolete
conffiles after package upgrades. Suggested by Michael Biebl.
* pre_remove_50_find_missing_copyright: Skip check for packages that are not
installed or have only config files remaining.
* post_remove_cleanup: Cleanup dovecot certificates.
* Add support for installing and upgrading to the multiarch ia32-libs* in
wheezy/sid on amd64. (Closes: #679780)
* Merge known problem analyses excessive_output and runtime_exceeded into
new known problem resource_violation.
* New known problem: Leaving obsolete conffiles after upgrade.
* New known problem: Modifying conffiles or other shipped files (issue).
* New bug template: partial-upgrade-file-overwrite.
[ Holger Levsen ]
* README.txt:
- reformat with shorter line lengths (Closes: #603873).
- fix some typos.
-- Holger Levsen Sat, 12 Jan 2013 12:12:45 +0100
piuparts (0.48) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Ignore *.dpkg-* conffile backups found in --scriptsdir directories.
- Report attributes (for reference and current file) in case of file
modification.
- Report paths of installed files that contain symlinks (for better
debugging of the problems caused by such a setup).
( test $(dirname $x) != $(readlink -f $(dirname $x)) )
- Add debsums, processes, and broken-symlinks check after the first purge
in --install-purge-install mode.
- Add --install-recommends option, disabled by default.
- Mark temporary directories with a .piuparts.tmpdir file.
- Use rm -rf --one-file-system to delete chroots.
* piuparts.conf:
- New per section settings: expire-old-days, expire-fail-days (defaulting
to 0 which means expiration is disabled).
- Split and rename piuparts-cmd into piuparts-command ([global] setting
that should include all common flags) and piuparts-flags (for additional
section-specific flags).
- Rename the tempdir setting to tmpdir, this will be used for the --tmpdir
argument of piuparts, too.
* piuparts-master.py:
- Enable recycling before initializing the real package db.
- Enable logging by default and set default log-file to
$SECTION/master.log. Use "/dev/null" to disable logging.
* piuparts-slave.py:
- Retry sections with higher precedence more often.
- Construct the piuparts command line from the new config options
piuparts-command, piuparts-flags, and tmpdir.
* piuparts-report.py:
- Don't hardcode section names, always show all known_problem reports.
* detect_piuparts_issues: Catch the chroot running out of space.
* reschedule_oldest_logs: Implement logfile expiration. Delete logfiles that
are marked for recycling and are older than expire-{old,fail}-days. These
packages have not been retested because of some dependency problem making
the package currently untestable. Deleting the log will cause the package
to be moved to dependency-failed-testing or similar state.
expire-*-days needs to be greater than reschedule-*-days to enable this.
* slave_cleanup: Only remove directories containing a .piuparts.tmpdir file.
* New bug template: copyright_file_missing_after_upgrade.
* New known problem: Installing something over existing symlinks.
-- Holger Levsen Fri, 30 Nov 2012 12:12:09 +0100
piuparts (0.47) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Add to ignored_files/ignored_patterns:
+ /var/log/dbconfig-common/dbc.log
- Print a final message before terminating in the panic() handler.
- Add support for running debsums before package removal, enabled by
default. May be disabled via --no-debsums. (Closes: #687538)
- Add debsums as dependency.
* piuparts-slave.py:
- Detect incomplete piuparts output and ensure such tests fail.
* piuparts-report.py:
- Rewrite find_files_with_suffix(), handle disappearing files.
* New custom script pre_remove_50_find_missing_copyright: check for missing
/usr/share/doc/$package/copyright files. (Closes: #683685)
* Rename custom script post_install_find_bad_permissions to
pre_remove_50_find_bad_permissions to avoid early failures during upgrade
tests that may be fixed in a later version.
* Remove post_purge_dbconfig_cleanup custom script.
* reschedule_oldest_logs: Refactored.
- Fix duplicate variable name usage that sometimes prevented recycling.
- Support a [global] auto-reschedule=no setting.
* detect_well_known_errors: Unclassified logs may still be bugged.
* detect_piuparts_issues: Detect incomplete logfiles.
* New bug template: modifies_conffiles.
* New known problem: Modifying conffiles or other shipped files.
* New custom script post_distupgrade_hack_debsums to work around #687611
which makes all debsum checks fail due to modification of
/usr/share/keyrings/debian-archive-removed-keys.gpg during upgrades.
-- Holger Levsen Fri, 28 Sep 2012 13:24:47 +0200
piuparts (0.46) unstable; urgency=low
[ Holger Levsen ]
* piuparts-report: drop in-code index.tpl stub.
* slave-join: fix bug preventing detection that the slave is not running.
* piuparts: call schroot with session: argument, thanks to Stephan Sürken
for the patch. (Closes: #686360)
[ Andreas Beckmann ]
* piuparts.py:
- Add to ignored_files/ignored_patterns:
+ /var/lib/cdebconf/{questions,templates}.dat-old
+ /var/lib/onioncat/
+ /var/lib/vmm/(./.*)? (Closes: #682184)
- Fix some leftover files and directories getting incorrectly accounted to
the package being tested instead of the dependencies in --warn-on-others
mode.
- Implement --install-purge-install option: test purging with all
dependencies still installed; re-install after purge.
- Install the dependencies and snapshot the chroot before the
--install-purge-install test. Check that the chroot after purge matches
the snapshot. (Closes: #668713)
* piupartslib/dependencyparser.py:
- Package names may now have a ":any" qualification in some cases. Extend
the parser to recognize and discard this suffix.
* piupartslib/packagesdb.py:
- Implement logfile recycling mode that avoids getting a large backlog of
untested packages due to uncoordinated log deletion. Delay log deletion
and give preference to packages (or versions) that have not yet been
tested. (Closes: #670150)
+ logfiles to be recycled should be hardlinked from pass/, fail/, ... to
recycle/
+ recycling can be enabled if recycle/ is not empty
+ treat packages with logfile in recycle/ as initially "unknown" and
compute state ("waiting[-for-dependency]-to-be-tested")
+ delete old logfile while reserving such a package
* piuparts-master.py:
- Accept and discard duplicate log submissions, preventing the slave from
retrying infinitely.
- Delay loading the Packages file which is only needed for "status" and
"reserve" commands, but not for submitting logs.
- Add "recycle" command to enter logfile recycling mode (needs to be
issued before "status" or "reserve").
- Remember idle state by creating stamp files.
- Add "idle" command to check a remembered idle status before trying to
"reserve" packages (avoid package state computation). Idle status (i.e.
no packages available for "reserve") will be remembered for an hour, but
any log modification (submission, deletion, marking for recycling) will
clear the idle status.
* piuparts-slave.py:
- Fix stopping the currently running test (Ctrl-C Ctrl-C).
- Handle master communication exceptions while sending logs or reserving
packages: skip the section but keep the slave running.
- Flush finished logs from sections that have processing postponed due to
precedence settings.
- Skip connection to master if we have reserved but no submittable logs.
- Submit finished logs after completing all reserved logs in a section.
- Send finished logs to master if interrupted with a single Ctrl-C.
- Try to unreserve all reserved logs after interrupt.
- Add SIGHUP handler that triggers flushing finished logs. Flushing (all
sections) will be done after the current test has finished.
- Enter logfile recycling mode if no section has packages left for regular
processing. Recycle logfiles until a section becomes available for
regular processing. If no logfiles could be recycled, retry after an
hour.
- Issue "idle" command before "reserve" and skip status computation and
reserving if master is idle.
* Simplify running piuparts from GIT.
* Reorganize layout in the GIT repository to reduce path nesting and length.
* Makefile:
- Separate build and install targets.
- Adjust for new layout, cleanup, simplify.
* conf/piuparts.apache: Set DefaultType text/plain (for the logfiles).
* reschedule_oldest_logs: Hardlink old logfiles to recycle/ instead of
deleting them. Cleanup obsolete rescheduling requests.
* New helper script: master-bin/reclassify_bugged.
* New custom script post_setup_forbid_home: replace /home with a file to
prevent creating home directories there. (Closes: #677332)
* New custom script post_install_find_bad_permissions: look for world
writable directories without sticky bit set.
-- Holger Levsen Mon, 03 Sep 2012 15:28:04 +0200
piuparts (0.45) unstable; urgency=low
[ Holger Levsen ]
* Merge piatti.git into piuparts.git, so that piatti.git has become obsolete
now. Update documentation accordingly.
* Further split packaging into piuparts-master, piuparts-slave and
piuparts-common binary packages (Closes: #585861) and cleanup
dependencies.
* Update piuparts.NEWS about master+slave packages.
* Switch to debhelper8 packaging.
* debian/copyright: use versioned URI of the machine-readable copyright file,
thanks lintian.
* Move /etc/sudoers.d/piuparts to piuparts-slave package.
* piuparts-master: depend on tango-icon-theme and drop sunny.png and
weather-severe-alert.png and use links instead.
* Reviewed and merged/cherry-picked all of the following changes.
[ David Steele ]
* Remove debian/postinst as it was only dealing with lenny time area
configuration files.
* Added server configuration files for apache, sudo, and cron.
* Use dh_install, in preparation for multiple package build.
* Added packaging to create the piuparts-server package out of the old
piatti directory tree (Closes: #652934).
- Create and configure piupartss and piupartsm users.
- Coordinate ssh keys between master and slave users.
- Create working directory tree for each under /var/lib/piuparts
* packagesdb.py:
- Calculate recursive reverse dependency count, and store in PackagesDB.
- Calculate the count of packages being blocked by a failed package.
* piuparts-report.py:
- Release Section object when section processing is complete.
(Closes: #673919)
- Display reverse dependency counts and block counts to failure summaries.
- Sort the failed-testing and cannot-be-tested summaries by block count.
(Closes: #674498)
- Replace O(n^2) search in remove_old_logs() with a hash
piuparts-report run time improved 20% on mature environment.
- Another search fix in create_package_summaries (find_links_to_logs)
yielding a 10x speedup for piuparts-report runs.
(Closes: #674672)
- Cleanup/wrap some long source lines.
- Cleanup/wrap some long html lines.
- Allow the html root to be changed from /
(piuparts.conf: e.g. [global] doc-root = /piuparts/).
[ Andreas Beckmann ]
* piuparts.py:
- Implement --install-remove-install option to test installation in
config-files-remaining state. (Closes: #359154)
- Report leftover symlinks with target.
- Report leftover directories with a trailing slash.
- Allow ignore entries to specifically match directories.
- Ignore patterns need to match the full filename, not a substring.
- Cleanup and regroup ignore lists.
+ Remove many obsolete entries not needed for squeeze or later.
+ Group ignore entries into piuparts state, system state, package
management, logfiles, home directories, application data, and HACKS.
- Add to ignored_files/ignored_patterns:
+ Default system logfiles as created by syslogd in /var/log/
+ Common empty directories in /etc not removed due to dpkg bug #316521
+ cdebconf status files
+ /boot/grub/
+ /etc/X11/default-display-manager
+ /etc/aliases.db
+ /etc/apt/trusted.gpg.d/.*.gpg~
+ /etc/network/interfaces
+ /etc/news/whoami
+ /usr/share/keyrings/debian-archive-removed-keys.gpg~
+ /var/crash/
+ /var/lib/citadel/(data/.*)?
+ /var/lib/gozerbot/
+ /var/lib/nagios/ (Closes: #668756)
+ /var/lib/openvswitch/(pki/.*)?
+ /var/spool/powerdns/ (Closes: #531134, #531135)
- Implement a timeout for commands being run in the chroot.
- Set time limit to 30 minutes per command. (Closes: #654423)
- Terminate all processes running inside the chroot before removing the
chroot; also in case piuparts aborts due to an error.
- Continue killing running processes as long as new processes get spawned
in the chroot.
- Perform chroot cleanup after receiving Ctrl-C/SIGINT, too.
- Let Chroot register/de-register the cleanup panic handler.
- Remove temp_tgz on error exit.
- Remove metapackage build directory on error exit. (Closes: #663702)
- Don't remove eatmydata when minimizing a chroot. (Closes: #658962)
- Add support for version-qualified package arguments (--apt foo=1.2-3).
- Switch fallback mirror to http://cdn.debian.net/debian.
* piuparts.conf:
- Defaults for all [section] settings can be set in the [global] section.
- Change master-command to not include the section name so that it can be
set in the [global] section.
The section name will be given as an argument to this command.
- Make doc-root default to "/".
- New per section key: area (set to one of main, contrib, non-free).
- New defaults: mirror=http://cdn.debian.net/debian, area=main.
- Compute URLs from mirror, distro, area, arch.
- Removed deprecated keys: known_circular_depends, packages-url,
sources-url.
* master/db/report: Add a new category /affected/ for failed logs where the
bug is in a dependency, but only exposed by the package being tested.
- In the BTS these are reported as (Closes: #657740)
Package: buggy-package
Version: buggy-version
Affects: package-under-test
Found: package-under-test/tested-version
- Reschedule /affected/ logs like /fail/ logs.
* piupartslib/conf.py:
- Add support for getting default values from a special section.
- Add methods to get mirror, distro, area, arch from config file or
defaults.
- Add methods to compute URLs to Packages and Sources files.
* piupartslib/packagesdb.py:
- Do not consider a package as "untestable" if untestable/ contains a
logfile from an obsolete version of that package.
- Simplify handling of package and dependency states.
- Do not inherit 'dependency-does-not-exist' state (which may be fixed
in the package) but propagate it to 'dependency-cannot-be-tested'.
- Remove 'no-dependency-from-alternatives-exists' state and use
'dependency-does-not-exist' instead.
- Remove 'unknown-preferred-alternative' state as it interferes with
circular dependencies involving alternatives and virtual packages.
- Fix state resolution of package dependencies containing alternatives,
virtual packages and circular dependencies.
- Remove 'circular-dependency' state.
- Stop using static known_circular_depends.
- Compute dependency cycles and test packages on such cycles if all
non-circular dependencies are satisfied.
- Consider any combination of two error states for the blocking count.
- Always run create_subdirs() during initialization.
* piuparts-master.py:
- Remove known_circular_depends handling.
- Reduce logfile verboseness: do not include received logs.
- Always chdir to master_directory, do not rely on being run from there.
- Record timestamps of submitted logs in submissions.txt.
* piuparts-slave.py:
- Randomize waiting time (between 60 and 180 seconds) if master is busy.
- Sleep until the next section can be tried, but at least 1 minute.
- Simplify and merge Section.setup() into Section.__init__().
- Generate master command: use global setting and append section.
- Test the 'dpkg' package for creating/updating a base_tgz.
- Don't update the tarball for disabled sections.
- Rewrite starting piuparts.
- Rewrite stopping piuparts for more reliable cleanup.
- Introduce Ctrl-C/SIGINT handling:
press Ctrl-C once to exit after the current piuparts test has finished,
press Ctrl-C again to cleanly terminate the currently running test,
press Ctrl-C again to abort any ongoing cleanup.
- Pass a version qualified package name (foo=1.2-3) to piuparts to ensure
the correct package version gets tested.
- Increase verboseness for untestable packages.
- Remove idle.stamp functionality.
* piuparts-report.py:
- Remove known_circular_depends handling.
- Exclude obsolete states from generated report.
- Establish packagesdb working directory in Section.
- Handle logs in /affected/ directories.
- detect_well_known_errors: Add bug links if bugs are known.
- New known problem: "packages have been kept back".
- Report rdeps and blocking counts in all error state reports.
- Fix another race with logfiles disappearing while copying.
* piuparts-analyze.py:
- Don't report further bugs/versions if we found a match.
- Classify logs with bugs filed into /bugged/ or /affected/.
- Write .bug files with links to the BTS.
* detect_archive_issues: Only consider logs older than one day for marking
as "untestable" to allow piuparts-analyze to handle the log first.
* Replace all references to lenny and squeeze with squeeze and wheezy.
* Add/update several bug reporting templates.
* Add custom scripts to aid debugging common problems in maintainer scripts.
* Add custom script for squeeze to ensure adduser, ucf, and install-info
are not purged.
* Add custom scripts to enable/perform additional cleanup procedures to
reduce the "FAIL: Package purging left files on system" errors if
--warn-on-leftovers-after-purge is not used.
(Closes: #668752)
* Ship custom scripts in the piuparts package.
* Comment sudoers and crontabs shipped in the master/slave packages,
requiring explicit editing after installation to activate them.
* Bump debhelper and dpkg dependencies for using dpkg-maintscript-helper.
* Remove obsolete /etc/piuparts/piuparts.conf from old piuparts package.
* Ship /etc/piuparts/piuparts.conf in the piuparts-slave package.
* Create missing home directories if the piupartsm/piupartss users remain
from a previously purged installation.
[ Leo Iannacone ]
* piuparts.py: add --existing-chroot option, specify a directory as source
for the chroot, instead of building a new one with debootstrap or
decompressing an archived one. (Closes: #658044)
[ Mika Pflüger ]
* piuparts.py: Remove obsolete functions shellquote, apt_get_knows, and
append_to_file.
[ Mehdi Abaakouk ]
* piuparts.py: Add support for schroot. (Closes: #530733)
-- Holger Levsen Thu, 21 Jun 2012 22:17:26 +0200
piuparts (0.44) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Add to self.ignored_patterns: /var/lib/ucf/.*
- Increase output limit to 3 MB (for dist-upgrading daptup).
- Do not miss Pre-Depends in --warn-on-others mode.
- Remove packages and reinstall missing ones in one apt-get call.
- Run 'apt-get dist-upgrade' (instead of upgrade) during chroot setup.
* piupartslib/packagesdb.py:
- Only consider the newest version of a package as test candidate.
(Packages files may contain more than version of the same package.)
* piuparts-master.py:
- Implement simple section locking to allow concurrent master instances
but allow only one instance per section.
* piuparts-slave.py:
- Do not leak the logfile file descriptor to piuparts.
- Skip disabled sections (max-reserved = 0).
- Do not abort if connection to master failed, just skip the section.
- Do not abort if Packages file cannot be fetched, just skip the section.
- Implement simple section locking to allow concurrent slave instances.
- Sections wait before retry 15 minutes after error and one hour if idle.
- Create idle.stamp files to remember idle sections for one hour and
notify concurrent slaves about idle state.
- Skip misconfigured sections that have neither 'distro' nor
'upgrade-test-distros' in their config. (Closes: #668863)
* piuparts-report.py:
- Add link to the list of untestable logs.
- Add more known problem reports: forgotten alternatives, warnings in
passed logs: leftover files and failed maintainer scripts.
- Do not abort if logfiles disappear during copying.
* Makefile: Install documentation with mode 0644 instead of 0755.
[ Holger Levsen ]
* piuparts-report: don't create output for disabled sections. (Bye bye
Lenny!)
* Bump Standards-Version to 3.9.3, no changes necessary.
-- Holger Levsen Thu, 17 May 2012 14:36:53 +0200
piuparts (0.43) unstable; urgency=low
[ Andreas Beckmann ]
* piuparts.py:
- Add to self.ignored_patterns: /run/.*
- Unset DISPLAY variable before running commands in the chroot.
- Read possibly remaining buffered output after a command terminated.
- Run apt-cache policy on the packages to be installed.
- Simplify keeping the reference chroot state: move tree info, selections
and diversions into a dict, can be extended easily.
- Fix --*end-meta documentation and actually enable these options.
- Cleanup options and settings.
* piupartslib/packagesdb.py:
- Restrict circular-dependency state to packages on the circle.
* piuparts-slave.py:
- Fix chdir in/out of the slave dir to work with relative paths.
* piuparts-report.py:
- Reorder the list of known problem analyses and add a few new ones.
- Correctly split + strip comma-separated lists.
* piuparts-analyze.py:
- Also consider bugs that 'affects' a package.
- Treat bugs without 'found' version as older than anything.
* Makefile: Support version number substitution in piuparts-*.py, too.
* README.txt: Document piuparts.conf settings used by piatti.git scripts.
[ Holger Levsen ]
* piuparts.1.txt: correctly reverse the description of --save-end-meta
and --end-meta.
* piuparts-report.py:
- add support for more static pages and add bug_howto.tpl, explaining how
to file meaningful bugs piuparts-analyse understands. This page also
links to the templates used for bug filing.
* piuparts-slave.py: make status message more useful by including the
section.
-- Holger Levsen Sun, 22 Jan 2012 13:03:59 +0100
piuparts (0.42) unstable; urgency=low
[ Holger Levsen ]
* piuparts.py:
- add to self.ignored_files: /etc/blkid.tab (Closes: #638831)
- add to self.ignored_patterns: /var/lib/apt/lists/.*
- apply patch by Gregor Herrmann to fix --minimize. (Closes: #648423)
* Remove Debian.NEWS entry about source in git. (Closes: #640121)
* piuparts.py, piuparts-report.py, ChangeLog: Expand tabs to spaces.
* Remove whitespaces from whitespace-only lines.
* piuparts-report:
- create maintainer subdirs if they don't exist.
- detect tests being terminated due to excessive output.
* Add git to Build-Depends-Indep: as git describe is used to generate
part of the version string for development builds.
* Add debian/.gitignore
[ Mika Pflüger ]
* piuparts-analyze.py:
- Rewrite to use python-debianbts to analyze if bugs are filed already.
- The BTS only tracks source versions, so remove binNMU-part from
package versions when comparing with versions from the BTS.
- Reduce noise in the output by only printing one action/advise per
package.
- Fix extraction of package versions from bug reports. Thanks to
Andreas Beckmann for catching and solving the error.
* debian/control: Add python-apt and python-debianbts to piuparts depends.
[ Scott Schaefer ]
* debian/copyright: Make it compliant with DEP-5.
* piuparts-slave.py:
- Replace deprecated os.popen2 with subprocess.Popen. (Closes: #640646)
- Add some more logging.
- Kill children (hard-coded value, 45 minutes) to ensure test doesn't
run "forever" (Closes: #640647, #387428)
* piuparty.py:
- Kill leftover processes (via SIGTERM, then if that fails, via SIGKILL).
(Closes: #522918)
- Test for incorrect diversion handling: (Closes: #588313)
a) Existing diversions removed/modified, and/or
b) Installed diversions not removed by purge.
* piupartslib/packagesdb.py: Modify several functions in PackagesDB class
to use python 'set' type, instead of a list. This permitted replacing
inefficient combination of unique() function and random.shuffle() with
python set.pop() method. Since python prohibits storing non-hashable
object in a set, minor modifications to piuparts-report and to
piuparts-master required. (Closes: #640648)
[ Andreas Beckmann ]
* *.py: Add vim modeline.
* piuparts.py:
- Add unittests for misbehaving symlinks.
- Fix resolving absolute symlinks of intermediate directory components,
i.e. /var/run -> /run while checking /etc/motd -> /var/run/motd.
Solves about 30000 false positives of
'Broken symlinks: /etc/motd -> /var/run/motd'. (Closes: #648784)
- When running commands in the chroot, redirect stdin from /dev/null.
- Stop using Popen.communicate() as it may run out of memory.
- Terminate commands producing more than 2 MB of output. (Closes: #649335)
- Create /etc/dpkg/dpkg.cfg.d/ if missing inside the chroot (e.g. on
lenny). (Closes: #647752)
- Remove logrotate and its dependencies only once.
- Only run 'apt-get update' after updating the sources.list.
- Only run 'apt-get clean' before creating tarballs or saving meta data.
- Do the same checks for running processes and broken symlinks in all
tests. (Closes: #648708)
- Create piupart's apt config in the chroot as /etc/apt.conf.d/piuparts
instead of /etc/apt.conf in order to allow overriding the settings from
custom scripts by just dropping new config bits in e.g.
/etc/apt/apt.conf.d/piuparts-foobar.
- Integrate diversion checks with Chroot.check_result().
- Use 'apt-get remove' to remove all packages at once with proper
dependency ordering. (Closes: #648162)
- Purge all packages at once instead of doing it one-by-one.
- Make restore_selections() reinstall missing packages. (Closes: #648707)
- Set more environment variables to let custom scripts know where and
when they are being run: PIUPARTS_TEST, PIUPARTS_PHASE,
PIUPARTS_DISTRIBUTION{,_PREV,_NEXT}. See the README for details.
(Closes: #589498)
- Add missing post_install_* custom scripts to install_packages_by_name().
(Closes: #628077)
- Remove pre_upgrade_* custom scripts, they can be replaced with
pre_install_* scripts that check for PIUPARTS_PHASE=upgrade.
- Add pre_test_* custom scripts. These are run exactly once at the
beginning of each test (after recording the initial chroot state).
- Allow multiple --scriptsdir options, the contents will be merged.
- Exclude /tmp/scripts when creating a tarball.
- Use --one-file-system when creating a tarball to exclude bindmounts etc.
- Restore base system from temp_tgz instead of running debootstrap again.
(Closes: #648193)
- Do not fail upgrade/distupgrade tests for a set of packages where not
all packages exist in the start distribution. This happens e.g. when
testing .changes files and packages were split/added. Only install the
old packages that actually exist according to 'apt-cache show'.
- Add --extra-old-packages option to intall additional old packages that
are not in the set of new packages to be tested. Useful for testing
upgrades with Conflicts/Replaces relationships, e.g. in package renames
or merges.
- Use consistent variable names for package lists. (Closes: #648177)
- Compute the changes in restore_selections().
- Check for settings.scriptsdir inside run_scripts().
- Consistently use chroot.relative() to build filenames inside the chroot.
* piupartslib/packagesdb.py:
- Handle 'unknown-preferred-alternative' state correctly.
- Add 'does-not-exist' state for dependency resolution to distinguish this
from 'unknown' state so that the latter only indicates 'unresolvable' or
'not yet resolved'.
- Handle virtual packages separately from real packages.
- Use get_package_state() internally which 'resolves' (purely) virtual
packages by default (can be disabled).
* piuparts-master.py:
- Add a 'status' command that reports package counts for all states.
* piuparts-slave.py:
- Fix triggering tarball recreation.
- Check tarball age regularily.
- Log tarball creation in *.tgz.log.
- Request and print package counts from master.
- Reload section config every time a section is being run.
- Add precedence attribute to allow prioritizing different sections and to
suspend processing of low priority ones while there are packages waiting
in more important sections.
* piuparts-report.py:
- state-*.html: Sort package lists by name, display state of all
alternative dependencies and packages providing a virtual dependency.
- source/?/*.html: Sort binary packages by name.
- maintainer/?/*.html: Sort source packages by name.
- Update list of error states to be highlighted.
- Archive logs of packages that were removed from the distribution.
- Speedup generating maintainer summaries.
* Makefile: Use 'git describe' to get an exact stamp for development
versions.
[ Dave Steele ]
* piuparts-slave.py: make Section.run() report the number of packages
processed and use this to decide whether a slave should sleep.
(Closes: #649967)
[ Stefano Rivera ]
* piuparts.py:
- Properly install and remove logrotate. (Closes: #638832)
- Use eatmydata by default, add option --no-eatmydata. (This was discussed
in #633033.)
-- Holger Levsen Fri, 23 Dec 2011 10:51:28 +0100
piuparts (0.41) unstable; urgency=low
* piupartslib/packagesdb.py: Apply patch by Scott Schaefer to also consider
alternative depends: the algorithm used to select the "best" alternative
is:
1) Prefer first alternative in state "essential-required"
2) If no "essential-required" alternatives, prefer first alternative
in state "successfully-tested"
3) Otherwise, prefer first alternative in state
"waiting-to-be-tested" IF NO REMAINING alternatives are in one of the
"unknown/fail" states
For this two new states have been introduced:
a) "unknown-preferred-alternative": equivalent of "unknown", this defers
calculation of this package's state, since one or more of its
alternative depends are "unknown" (or
"unknown-preferred-alternative"), and no alternative is either
"essential-required" or "successfully-tested". The alternatives will
be re-tested on subsequest pass.
b) "no-dependency-from-alternatives-exists": if none of the alternatives
can be found in the archive.
(Closes: #526045)
* piuparts-report.py:
- correct a typo from the patch for #523950.
- make_stats_graph: draw the two new states as well.
- point to source code in git instead of svn.
* piuparts.py:
- use proxy settings either from apt configuration or http_proxy
environment variable, the latter overwriting the former (if present)
- Thanks to Scott Schaefer for the patch. (Closes: #632046)
- new option "--no-install-purge-test" to only do upgrade tests
- Thanks to Andreas Beckmann for the patch (Closes: #588482)
- run dpkg with --force-unsafe-io by default and introduce new option
"--dpkg-noforce-unsafe-io" to disable this feature. (Closes: #633033)
Thanks to Scott once more!
- new option: "--fail-on-broken-symlinks". Remove option "--warn-symlinks"
as this is the default now.
* piuparts-slave:
- check if chroot-tgz is older than max-tgz-age (defaulting to 30 days)
and recreate it, if it is. Keep backup and put in back in place when
debootstrapping a new chroot-tgz fails. Retry after min-tgz-retry-delay
seconds. Thanks to Scott Schaefer for the patch. (Closes: #632924)
- document new configuration options max-tgz-age and min-tgz-retry-delay
in README.txt.
- fix bug in piuparts-slave which prevented running a single section by
including section name as command-line argument - thanks again to Scott
and see 632924 too.
* debian/control:
- require python >= 2.6.6-3~, get rid of python-support build-depends.
- add X-Python-Version: pseudo-header
* debian/rules: use dh_python2 instead of python-support to build the
package
* Makefile: build for python 2.6 and 2.7 instead of 2.5 and 2.6.
* Source code is now stored in git.
http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git
git clone git+ssh://git.debian.org/git/piuparts/piuparts.git
Thanks to Thomas Koch for setting those up and keeping history!
* The configuration and scripts used on piatti.debian.org have also been
moved to git, the URLs are the same, just s#piuparts#piatti#.
* Add entry about git to NEWS.
* Update Vcs-*-Headers in control to reflect move to git.
-- Holger Levsen Wed, 24 Aug 2011 13:47:07 +0200
piuparts (0.40) unstable; urgency=low
* piuparts.py:
- make "natty" the default Ubuntu distribution.
- add to self.ignored_files:
- /var/log/bootstrap.log
* debian/control: depend on ${python:Depends} instead of just python. Thanks
to Jakub Wilk. (Closes: #614617)
* *.py: do not use Python built-in function names as variable name. Thanks
to Carl Chenet for a first patch and Scott Schaefer for the applied one.
(Closes: #523950)
* Bump Standards-Version to 3.9.2, no changes necessary.
* debian/rules: add empty build-arch and build-indep to appease lintian.
* Makefile: drop workaround for #559305 in lenny.
-- Holger Levsen Sat, 25 Jun 2011 21:20:31 +0000
piuparts (0.39) unstable; urgency=low
[ Holger Levsen ]
* The vintage of 2010 release!
* piuparts-report.py:
- report packages which failed
- due to broken maintainer scripts,
- due to output from cron after removal,
- due to not enough force being,
- due to a problem with pre-depends,
- due to files having been modified after purge,
- due to files having disappeared after purge and
- due to problems configuring a database.
- only report about (un)owned files and directories and symlink issues in
sid - that's hardcoded as it's not practical to track them anywhere else
anyway. And at least for broken symlinks this will also stay like this
for a long time.
- visual improvements in the output.
- ignore udebs and binaries from other archs too.
- graph is generated using all the available data points.
* piupartslib/packagesdb.py:
- automatically break circular dependencies when there are only circular
dependencies left, using a configurable list of circular depdencies in
piuparts.conf (Closes: #526046)
The list of circular depends is taken from a script written by Robert
Lemmen available at http://debian.semistable.com/debgraph.out.html -
obviously it would be better to merge this into piuparts directly.
Note that the dependency resovler is still buggy, the state
waiting-for-dependency-to-be-tested still contains some packages
with failed depends (due to them being dependent on packages with
circular depends). This bug has no effect other than that (so the state
waiting-to-be-tested is calculated correctly). This bug is also no
regression.
- get rid of the longtime unused states "fixed" and "fix-not-yet-tested"
(also in piuparts-master and piuparts-report).
- forget reservations of untestable packages.
* piuparts-slave.py:
- fix crash when going into sleep when idle.
- add support for only doing upgrade tests (without the basic test in the
upgrade target distro).
- honor mirror configuration option also for upgrade-tests.
- also do upgrade tests if the version of a package being tested (the one
in the upgraded distro) is not available in the distro being upgraded
from. (This is a very short test, but a successfully one.)
- make configuration setting "distro" to not default to any distro.
* piuparts.py:
- new option: --log-level to specify the verbosity of piuparts
output. Thanks to Fabrice Coutadeur for the patch! (Closes: #567190)
- new option: --warn-on-leftovers-after-purge to not fail if a package
leaves files behind after purge. (Closes: #566599)
- add to self.ignored_files:
- /var/lib/apt/lists/partial/.delete-me-later
- /var/log/alternatives.log (Closes: #591966)
- /var/log/apt/history.log (Closes: #594964)
- /usr/share/fonts/X11/misc/fonts.alias (Closes: #576321)
- add to self.ignored_patterns:
- /etc/init.d/.depend.*
- /var/mail(/.*)? - this used to be just /var/mail/.*
- /var/lib//update-rc.d(/.*)? (Closes: #605127)
- create temporary DEBIAN directory for the piuparts-depends-dummy package
with 0755 perms regardless of the umask of the calling shell.
(Closes: #573904)
- fix --do-not-verify-signatures option (Closes: #574936) - thanks to
Cleto Martin Angelina for the patch.
- make Chroot.diff_meta_data() special case and ignore files matching
(/etc/rc.\.d/)[SK][0-9]{2}(.*)$ so that piuparts doesn't complain about
initscripts renamed by insserv. Thanks to by Andreas Beckmann for the
patch! (Closes: #586793)
- remove logrotate and depended packages after the test. (Closes: #602409)
This is a fix for the incomplete patch for #566597. Again, this is a
hardcoded list. :-(
- new custom script: pre_remove_ (Closes: #539146)
- set environment variable PIUPARTS_OBJECTS in custom scripts to a space
separated list of packages / changes files being tested.
- do not call apt-get with --no-remove when installing packages.
(Closes: #603453)
- apply patch by Andres Mejia to fix parsing of the --force-confdef option
and also to configure apt to use that option. Thanks, Andres.
(Closes: #605475)
- don't panic if package is not known by apt-get, this probably just means
a package not yet in the archive is being tested. (Closes: #566672)
Thanks to Cleto Martin Angelina for the patch!
- fix parsing of the options --end-meta and --save-end-meta.
- supply help texts for --end-meta and --save-end-meta options, also add
comments to install_and_upgrade_between_distros() to make the code
easier to understand and hopefully improve in future. (Closes: #560050).
- add logging to install_and_upgrade_between_distros() to point people to
read the functions source code if they wonder why the log (for distro
upgrade tests) looks like it looks.
* piuparts.1.txt:
- update the pointer to custom-scripts.txt to point to README.txt, as
those two files have been merged. Thanks to Gregor Hermann for
spotting this. (Closes: #574504)
- indicate that it's possible to check several packages or .changes files
at once.
- update several options descritpions.
* README.txt
- better documentation of custom scripts.
- reformat to achieve consistent width.
* Add ${misc:Depends} to Depends in debian/control.
* Remove versioned dependencies on debootstrap as the version available
in Etch is sufficient. Drop build-depends on dpkg-dev as it's in
build-essential.
* Build-depend on debhelper>=7 and use dh_prep instead of dh_clean -k, bump
compat level to 7.
* Bump Standards-Version to 3.9.1, no changes necessary.
* Remove Ian Jackson from uploaders - thanks for your work, Ian!
* Makefile: support python 2.5 and 2.6 instead of 2.4 and 2.5, adjust
debian/control accordingly.
* debian/preinst: update code snipplet rm_conffile() from
http://wiki.debian.org/DpkgConffileHandling
[ Evgeni Golov ]
* piuparts.py:
- Check for remaining logrotate files which produce output after the
package was removed. (Closes: #566597)
[ John Wright ]
* Use debian module when available, else debian_bundle module.
(Closes: #586443)
* Use built-in set type, available since python-2.4.
-- Holger Levsen Tue, 04 Jan 2011 14:12:30 +0100
piuparts (0.38) unstable; urgency=low
* piuparts.py:
- Add support for using LVM snapshots. Thanks to
Patrick Schoenfeld for the patch. (Closes: #559449)
- Add support for warning on broken symlinks. Thanks to Carl Chenet for
the patch. (Closes: #440151)
- Make --help usuable without needing to be run as root. (Closes: #547379)
- --skip-minimize is now the default. This is because debootstrap is called
with --variant=minbase instead of --resolv-deps now and because if a
base.tgz or an lvm snapshot is specified, piuparts should not touch it.
(Closes: #539142, #559048)
Introduce a new option, --minimize, to minimize a chroot with debfoster.
(This is the old default behaviour. In future versions the --minimize
option might be removed again.)
- Add two new options: --keyring to specify the keyring to use with
debootstrap (defaults to /usr/share/keyrings/debian-archive-keyring.gpg)
and --do-not-verify-signatures to not use --keyring with debootstrap.
(Closes: #545907)
In the chroots, APT::Get::AllowUnauthenticated is set accordingly in
/etc/apt/apt.conf.
- Add new option, --pedantic-purge-test, to tell piuparts to be pedantic
when checking if a purged package leaves files behind. If this option is
not set, files left in /tmp are ignored. (Closes: #528266)
- Add new option, --dpkg-force-confdef, to make dpkg use --force-confdev,
which lets dpkg always choose the default action when a modified conffile
is found. This options will make piuparts ignore errors it was designed
to report and therefore should only be used to hide problems in
dependent packages. (Closes: #466118)
- Add another type of custom-scripts, post_setup_, which are executed after
the chroot was setup. (Closes: #466043)
- Create policy-rc.d in 2nd chroot when using -b. (Closes: #466049)
Thanks again to Patrick Schoenfeld for his help.
* piuparts-report: report broken symlinks.
* piuparts-slave:
- gracefully deal with upgrade-test-distros and upgrade-test-chroot-tgz
not being set in the configuration. Thanks to Carl Chenet and Patrick
Schoenfeld for the patch. (Closes: #519192)
- new default value for idle-sleep, 300 instead of 10 seconds. Also the
slave will now only sleep if there is nothing to do for any section.
* Makefile: workaround 559305 by calling a2x twice.
* Update debian/NEWS.
* Replace all references to sarge and etch with ones to lenny and squeeze.
-- Holger Levsen Thu, 17 Dec 2009 14:10:27 +0000
piuparts (0.37) unstable; urgency=low
* piuparts-report.py: report packages with update-rc.d warnings and those
failing due to insserv errors. (Closes: #546540)
* Fix typo in NEWS.Debian, thanks to Justin Rye for spotting it.
(Closes: #547439)
* piuparts.py:
- added to self.ignored_patterns:
- /var/cache/man(/.*)? (Closes: #550953)
- /var/lib/insserv/run.*.log
- /var/lib/mercurial-server(/.*)?
- removed from self.ignored_files:
- /var/cache/man/index.db
* fix typo in -i option in manpage (Closes: #555202), thanks to James Vega
for spotting it.
* Make "lucid" the default Ubuntu distribution (Closes: #559047), thanks to
Krzysztof Klimonda.
* fix FTBFS by adding "-r /etc/asciidoc" to the a2x call in Makefile, thanks
to Michael Vogt. (Closes: #559299)
-- Holger Levsen Thu, 03 Dec 2009 13:25:43 +0000
piuparts (0.36) unstable; urgency=low
[ Holger Levsen ]
* The "For me. For you. For everybody." release.
* Break backwards compatibility of the configuration files for master-slave-
mode. Merge those three into one: /etc/piuparts/piuparts/piuparts.conf.
Introduce a new global section in piuparts.conf which is shared among the
other sections there.
* piuparts.py:
- add check whether scriptsdir exits, to fail gracefully if not.
- copy scriptsdir to chroot also when doing upgrade tests.
- added to self.ignored_files:
- /etc/shadow and /etc/shadow- (just as /etc/passwd* and /etc/group*)
- /var/games (see #524461)
- /etc/apt/trusted.gpg and /etc/apt/trusted.gpg~ (just as
/etc/apt/trustdb.gpg and other backup files)
- added to self.ignored_patterns:
- /var/lib/dpkg/triggers/*
- /var/lib/ldap(/.*)? (see #527898)
- /var/backups/.*
- modified self.ignored_patters:
- ignore everything in /tmp, not just in /tmp/scripts (see #528266)
- configure apt in chroots to not install recommends and suggests.
- add support for scanning for packages in changes files, thanks to Andres
Mejia for the patch. (Closes: #352940)
- change some methods from using 'args' to 'package_list'. This more
accurately represents what is being passed into these methods now.
- add an optional parameter to panic() method to specify what exit status
to use. Also thanks to Andres.
- add advice how to read the logfile to the top of the logfiles produced.
- add "FAIL: " to logging output for seven more failure types, so that it
becomes easier to group failure causes.
- allow piuparts to be stopped with control-C. Thanks to Carl Chenet for
the patch. (Closes: #523958)
- fail gracefully if piuparts is run as non-root user. Thanks to Ignace
Mouzannar for the patch. (Closes: #538273)
* piupartslib/packagesdb.py:
- change the test whether a package is testable to check whether the
package is of priority "required", and not whether it's "Essential".
- rename status "essential-required-important" to "essential-required"
as important packages can be tested like all the others.
- handle binNMUs where the Source header in the Packages file includes the
Version and the Source package name in one line.
- fix get_packages_in_state() to only return unique packages and not also
the provided ones as exact copies.
* Install scripts for master-slave mode without .py extension.
* piuparts-master.py: remove code to write statistics, that is done only in
piuparts-report.py now.
* piuparts-slave.py:
- support looping trough multiple sections in piuparts.conf via new config
value "sections". Thanks to Frank Ploss for writing this patch with me
and helping me understand the code much better!
- allow the user to stop the slave with control-C.
- call piuparts with --mirror when mirror is set in piuparts.conf.
* Re-add piuparts-reports.py which got removed/lost between 0.14 and 0.20
without changelog entry.
* piuparts-report.py:
- support sections in configuration file.
- support looping trough multiple sections in piuparts.conf via new config
value "sections".
- generate counts.txt (per section) with raw numbers of packages per state,
include these stats in the debug output (so it gets included in the mails
send by cron.)
- introduce new setting "sources-url" for piuparts.conf.
- generate sources.txt (per section) with a summary status per source
package (for the PTS to display a source packages piuparts state)
- generate html status pages for all source packages (to link from the PTS)
with links to state explaination and available logfiles, handle udebs.
- provide links to logfiles in statistics page.
- provide links to source packages pages from state pages and back, as well
as links to the dependencies state.
- draw graphs of package states over time (if pypthon-rpy and gs are
installed) - thanks to Thomas Viehmann and Dirk Eddelbüttel for
inspiration and support.
- create maintainer centric pages for each email address found in
Maintainers or Uploaders.
- improve layout, generate navigation for all pages.
- use icons to provide a quick overview of a source packages status.
(Thanks to tango.freedesktop.org for the icons!)
- include index.tpl (if it exists) from output-directory into the
generated index page, so one can add news to the index page without
editing piuparts-report.py.
- generate pages for known issues, which are detected by bash helper
script, run by cron on piuparts.debian.org and available from
svn/piuparts/piatti/home/piupartsm/bin/detect_well_known_errors
* Always use alphabetic time zone abbreviation in timestamps.
* Makefile:
- add "~$date" to versionstring if building an unreleased version, thus
adding dpkg-dev to build-depends.
- split install target into install-doc, install-conf and install, to aid
development and deployment from trunk.
* debian/control:
- depend on python (>>2.4), make dependency to python-debian unversioned.
- add build-dependencies on debhelper, asciidoc and xmlto.
- dependency gs has been renamed to ghostcript
- remove build-dependencies on docbook2x and docbook-xml.
- suggest python-rpy and gs to improve piuparts-report output.
- set Homepage: to http://piuparts.debian.org
- remove Lucas Nussbaum, Ana Guerrero, John Wright and Bill Allombert from
uploaders - thanks for your work!
- Replace Ians email address with one that doesn't give an errorcode when
sending mail to it
- bump Standards-Version to 3.8.3, no changes necessary.
* Rewrite debian/rules from scratch using debhelper.
* Merge README, how-to-use-piuparts.txt and custom-scripts.txt into
README.txt, convert it to asciidoc and build pdf and html versions of it.
* Restructure and update README.txt to reflect the configuration changes in
master-slave mode.
* Add debian/NEWS file.
* Rewrite piuparts manpage in asciidoc.
* Build and install html version of the piuparts manpage.
* Update debian/copyright to reflect that piuparts is licenced under GPL2+.
* Update FSF address in all files referencing the GPL.
* Remove unused file piuparts.css.
[ Lars Wirzenius ]
* Removed Lars Wirzenius as uploader.
-- Holger Levsen Tue, 08 Sep 2009 14:35:42 +0200
piuparts (0.35) unstable; urgency=medium
* Fix recursive failure which occured when selinux-utils was installed but
not enabled. Thanks to Peter De Wachter for the patch. (Closes: #519017)
* Output path to temp directory if -k is used. (Closes: #466112)
-- Holger Levsen Tue, 10 Mar 2009 00:43:22 +0100
piuparts (0.34) unstable; urgency=low
[ Holger Levsen ]
* Mount /selinux on systems where selinux is enabled. Thanks to Filippo
Giunchedi for the patch! (CLoses: 507171)
* Remove wrong advice in debian/copyright which irritated lintian.
[ Filippo Giunchedi ]
* Check for any output when running cron files in addition to exit code
-- Holger Levsen Fri, 27 Feb 2009 12:34:31 +0100
piuparts (0.33) unstable; urgency=low
* Added --bindmount option, thanks to Aníbal Monsalve Salaza for the patch.
(Closes: #496186)
-- Holger Levsen Sat, 08 Nov 2008 17:07:22 +0000
piuparts (0.32) unstable; urgency=low
[ John Wright ]
* Fix a typo in how-to-use-piuparts.txt.
[ Holger Levsen ]
* Replace all references to sarge and etch with etch and lenny.
(Closes: #466111)
* Update README to reflect that piuparts runs fine in etch.
* Rename the post_upgrade custom script to post_distupgrade and introduce
pre_distupgrade.
* Bumped standards version, no changes needed.
-- Holger Levsen Fri, 17 Oct 2008 10:28:14 +0000
piuparts (0.31) unstable; urgency=low
[ Lars Wirzenius ]
* piuparts.docbook: Added a few words of description of the tarball
that -b expects. (Closes: 466048)
[ Lucas Nussbaum ]
* Added a --debfoster-options option, to allow the user to override
debfoster's default options and test with more packages installed
in the chroot. (Closes: #458922)
* Mention piuparts -s in in how-to-use-piuparts.txt.
* Finally fixes the check for broken symlinks. Thanks go to
Tobias Grimm for the patch. (Closes: #468157, 478587)
* Ignore /var/cache/ldconfig/aux-cache.
* Added myself to uploaders.
* Keep /proc mounted. Switch to calling tar directly in
pack_into_tgz(). Minor refactorings. Thanks go to Tobias Grimm
for the patch. (Closes: #478577)
* Move piuparts to priority: extra, since it depends on debootstrap which
is extra. (Closes: #477634)
[ Luk Claes ]
* Added -w to lsof call to suppress warnings. (Closes: #466102).
[ Holger Levsen ]
* Add a copyright statement to debian/copyright.
* Add support for post_upgrade custom script.
* Minor fixes in how-to-use-piuparts.txt.
* piuparts.docbook: Correct the path to dtd, it's
/usr/share/xml/docbook/schema/dtd/4.1.2/docbookx.dtd and add a
build-dependency for docbook-xml
-- Holger Levsen Sun, 11 May 2008 22:17:52 +0200
piuparts (0.30) unstable; urgency=low
* piuparts.py:
- Implement Chroot.create_temp_tgz_file() (since it's used in the VirtServ
subclass)
- Fix a typo -- chroot.create_temp_tgz() was being called instead of
chroot.create_temp_tgz_file() (Closes: #465416)
-- John Wright Thu, 14 Feb 2008 20:32:35 -0700
piuparts (0.29) unstable; urgency=low
[ Ana Beatriz Guerrero Lopez ]
* Updated uploaders, remove Amaya and Alastair.
[ Lars Wirzenius ]
* piuparts.py: Call "apt-get update" before calling Chroot.minimize, so
that we can find the debfoster package on Ubuntu.
* debian/control: Fixed "Uploaders:" to have my preferred e-mail address
(liw@iki.fi) instead of my Debian one.
* piuparts.py: Added -D option to set Debian flavor, plus two sets of
default settings depending on the flavor, one for Debian itself, and
another for Ubuntu. The settings choose default mirror and distribution
set (sid vs gutsy). This will allow an Ubuntu version of the package
to set defaults at package building time.
* piuparts.py: Report target of broken symlink correctly, instead of
saying "True".
* piuparts.py: Use lsb-release to guess the Debian flavor.
* debian/control: Added dependency on lsb-release.
* piuparts-master.py: Make the master write summary of total pass/fail
packages, plus status per binary package. This is for having the
Debian PTS and the Ubuntu developer weather report show summaries
of piuparts results.
* integraged changes from Ubuntu by Ian Jackson to support autopkgtest.
* wrapped long lines in debian/control.
[ Holger Levsen ]
* Added myself to uploaders.
* Added Homepage: header to debian/control and changed the homepage to be
http://wiki.debian.org/piuparts.
* Use Vcs-* headers in debian/control instead of XS-Vcs-*.
* Bumped policy version to 3.7.3.
* Updated download location in debian/copyright.
[ John Wright ]
* Change the --warn-on-others implementation to create a dummy metapackage
with the needed dependencies and install it in the chroot, rather than
using "apt-get install package package-". The previous implementation
had issues when the package was already installed (and especially if the
package had essential packages in its reverse dependencies). This has
the pleasant side-effect of making --warn-on-others work both with and
without --apt. (Closes: #458929)
* Fix a bug in check_for_broken_symlinks(): the targets for the broken
symlinks (used for logging) were being read from files on the real root
filesystem, rather than in the chroot.
-- Lars Wirzenius Fri, 01 Feb 2008 16:38:38 +0200
piuparts (0.28) unstable; urgency=low
* Fix an UnboundLocalError introduced in the --warn-on-others fix that would
occur if run without the --apt option. (Closes: #443641)
-- John Wright Sat, 22 Sep 2007 23:32:17 -0600
piuparts (0.27) unstable; urgency=low
* Add support for testing multiple distributions and architectures to
piuparts-master.py and piuparts-slave.py. Please see the README file,
piuparts-master.conf.sample and piuparts-slave.conf.sample for more
details. (Closes: #349365)
* Fix the --warn-on-others option. Now, dependencies are installed before
the packages we wish to test, and an inventory is taken then, so that we
can know which errors were caused by the packages explicitly specified on
the command-line. Currently, this requires --apt, and doesn't work quite
as advertised if there are circular dependencies with the packages you wish
to test (see the man page for more details). (Closes: #440310)
* debian/control:
- Update my email address in the Uploaders field
-- John Wright Sun, 16 Sep 2007 22:28:14 -0600
piuparts (0.26) unstable; urgency=low
* Update list of ignored files. (Closes: #439592)
* In the cron test, not executable files are not run. (Closes: #440141)
-- Ana Beatriz Guerrero Lopez Fri, 31 Aug 2007 15:44:36 +0200
piuparts (0.25) unstable; urgency=low
* Add the option to run custom scripts inside the piuparts chroot.
Scripts are stored in a directory and give it as argument with the
option --scriptsdir=/dir/with/the/scripts
This feature is still experimental :)
* Add custom-scripts.txt with information about the custom scripts.
-- Ana Beatriz Guerrero Lopez Tue, 28 Aug 2007 14:39:32 +0200
piuparts (0.24) unstable; urgency=low
* Add /var/lib/apt/extended_states to ignored_files. Thanks Anibal!
(Closes: #434980)
* Add quick howto about how to use piuparts under docs.
* Add test that checks the output from the cron files left in the system
after removing a package. This includes the option --skip-cronfiles-test,
to allow skipping this test.
-- Ana Beatriz Guerrero Lopez Mon, 20 Aug 2007 10:27:29 +0200
piuparts (0.23) unstable; urgency=low
* Cosmetic change, now file owner and file are shown in the same line.
* Add option --list-installed-files, that list files added/removed/modified
to the chroot after the package's installation. (Closes: #431821)
* Add option --no-upgrade-test, that allows skip testing upgrade from an
existing version in the archive. (Closes: #349933)
-- Ana Beatriz Guerrero Lopez Tue, 10 Jul 2007 19:47:21 +0200
piuparts (0.22) unstable; urgency=low
[ John Wright ]
* debian/control:
- Add XS-Vcs-Svn and XS-Vcs-Browser fields
[ Ana Beatriz Guerrero Lopez ]
* Rename piuparts.py.in back to piuparts.py.
* Add option --skip-minimize that allows skip the minimize chroot step
with debfoster.
* Remove m4 substitution, and use sed instead. Drop Build-Depends on m4.
(Closes: #431248)
* Drop support for python 2.3 and add support for python 2.5.
* debian/rules:
- Remove execution of checkversion in package build, not longer needed.
* debian/control:
- Add myself to uploaders.
- Add (future) piuparts website.
-- Ana Beatriz Guerrero Lopez Wed, 04 Jul 2007 21:03:44 +0200
piuparts (0.21) unstable; urgency=low
* Convert to Debian-native packaging style. From now on, changes to piuparts
will be tracked here. (Closes: #389610)
* Makefile:
- Give docbook2x-man a --encoding=utf-8 argument. It was failing to build.
- Generate piuparts.py from piuparts.py.in, filling in the VERSION variable
with the version from debian/changelog
* debian/control:
- Fix the name of the maintainer (we're the "piuparts developers team", not
Lustre)
- Correct my email address in the uploaders field
- Add Build-Depends on m4
* piuparts.py.in:
- Don't call shellquote() on arguments, since we're not using the shell.
(Closes: #386839)
- Add a --warn-on-others option. See the man page for details.
* piuparts.docbook:
- Document the --warn-on-others option.
-- John Wright Thu, 21 Jun 2007 00:27:22 +0100
piuparts (0.20-3) unstable; urgency=low
* New Maintainer(s): piuparts team. Closes: #390754.
-- Alastair McKinstry Mon, 23 Oct 2006 16:02:19 +0100
piuparts (0.20-2) unstable; urgency=low
* Orphaning.
-- Lars Wirzenius Mon, 2 Oct 2006 23:43:00 +0300
piuparts (0.20-1) unstable; urgency=low
* New upstream version. Fixed Debian bugs:
- Symlink correctness checking was broken when symlinks pointed at
symlinks. (Closes: #387796)
- fails if a deb filename contains a "strange" char. (Closes: #386839)
-- Lars Wirzenius Fri, 22 Sep 2006 12:58:24 +0300
piuparts (0.19-1) unstable; urgency=low
* New upstream version. No Debian bugs fixed.
-- Lars Wirzenius Fri, 8 Sep 2006 20:28:31 +0300
piuparts (0.18-1) unstable; urgency=low
* New upstream version. No Debian bugs fixed.
* debian/control: Updated for new Python policy.
* debian/prerm, debian/postrm: Written. They compile/remove byte code
files.
-- Lars Wirzenius Thu, 7 Sep 2006 20:42:03 +0300
piuparts (0.17-1) unstable; urgency=low
* debian/control: Added dependency on lsof, since that is now used to
check that there are no processes running inside the chroot.
* New upstream version, fixes the following open bugs in Debian:
- missing words in piuparts(1) (Closes: 362551)
- default ignore pattern for papercut files added (Closes: #355401)
-- Lars Wirzenius Sun, 25 Jun 2006 19:35:19 +0300
piuparts (0.16-1) unstable; urgency=low
* New upstream version. Fixes the following bugs reported against the
Debian package:
- TODO: keep track of bugs to show expected failures and unexpected
successes (Closes: #353215)
- It would be nice if the chroot were updated before the snapshot
(Closes: #356678)
* debian/control: Made the dependency on debootstrap be versioned.
Closes: #355875.
-- Lars Wirzenius Fri, 22 Mar 2006 22:40:00 +0200
piuparts (0.15-1) unstable; urgency=low
* New upstream version. Includes fixes for Debian bugs:
- "Specify packages to use for testing" (Closes: #354811)
- "Misleading documentation of -v" (Closes: #352941)
-- Lars Wirzenius Wed, 01 Mar 2006 20:53:00 +0200
piuparts (0.14-1) unstable; urgency=low
* New upstream version. Includes fix for:
- "piuparts should flag as an error a failed installation of a dependency"
(Closes: #343324)
-- Lars Wirzenius Sun, 18 Dec 2005 23:22:00 +0200
piuparts (0.13-1) unstable; urgency=low
* New upstream version.
* debian/changelog: Minor tweaking to the description.
-- Lars Wirzenius Sat, 12 Nov 2005 00:53:04 +0200
piuparts (0.12-1) unstable; urgency=low
* New upstream version. Fixes bug in Debian:
- dist-upgrade to experimental does not work (now documented as a
feature). Closes: #330749.
-- Lars Wirzenius Mon, 17 Oct 2005 21:03:12 +0300
piuparts (0.11-1) unstable; urgency=low
* New upstream version. Fixes bugs in Debian:
- Checks whether there are packages on the command line before creating
a chroot. Closes: #322441.
- apt-get configured to allow unauthenticated repositories.
Closes: #326705.
-- Lars Wirzenius Fri, 14 Oct 2005 01:16:25 +0300
piuparts (0.10-1) unstable; urgency=low
* New upstream version.
-- Lars Wirzenius Thu, 15 Sep 2005 23:31:30 +0300
piuparts (0.9-1) unstable; urgency=low
* New upstream version, fixes bugs reported against Debian:
- Upstream version number in Debian package and in upstream sources
now match. Closes: #326058.
- Files reported by piuparts are now associated with their packages,
when possible. Closes: #324248.
* debian/rules: Added "checkversion" target to make sure we no longer
do stupid things like #326058, at least in the Debian package.
-- Lars Wirzenius Fri, 09 Sep 2005 00:34:36 +0300
piuparts (0.8-1) unstable; urgency=low
* New upstream version, fixes bugs:
- Example in manual page now works (Closes: #319990)
- Temporary directories are now removed on failure (Closes: #323653)
- Parsing /etc/apt/sources.list now better (Closes: #319989)
- Temporary directorys are now put in $TMPDIR or /etc (Closes: #322440)
* debian/control: Added build-depends on docbook2x. Closes: #318693
* debian/changelog: Added note about upstream source repository.
-- Lars Wirzenius Mon, 29 Aug 2005 23:03:32 +0300
piuparts (0.7-1) unstable; urgency=low
* New upstream version.
* This is the first package that will be uploaded to the Debian archive.
-- Lars Wirzenius Fri, 15 Jul 2005 13:09:00 +0300
piuparts (0.6-1) unstable; urgency=low
* First release of the Debian package.
-- Lars Wirzenius Tue, 5 Jul 2005 20:08:00 +0300
piuparts-0.56/debian/piuparts-common.install 0000664 0000000 0000000 00000000075 12144650665 016165 0 ustar etc/piuparts/distros.conf
usr/lib/*
usr/share/piuparts/lib/*
piuparts-0.56/debian/piuparts-common.pyinstall 0000664 0000000 0000000 00000000226 12145710343 016523 0 ustar piupartslib/__init__.py piupartslib
piupartslib/conf.py piupartslib
piupartslib/dependencyparser.py piupartslib
piupartslib/packagesdb.py piupartslib
piuparts-0.56/debian/piuparts-master.examples 0000664 0000000 0000000 00000000077 12155061511 016326 0 ustar instances/piuparts.conf.anbe
instances/piuparts.conf.pejacevic
piuparts-0.56/debian/piuparts-slave.links 0000664 0000000 0000000 00000000204 12144650665 015453 0 ustar /usr/share/piuparts/slave/slave_run /usr/sbin/piuparts_slave_run
/usr/share/piuparts/slave/slave_join /usr/sbin/piuparts_slave_join
piuparts-0.56/debian/piuparts.NEWS 0000664 0000000 0000000 00000004022 12144650665 014001 0 ustar piuparts (0.45) unstable; urgency=low
For those interested in running their own local instance of
http://piuparts.debian.org there are now piuparts-master and
piuparts-slave packages for ease of installation.
This is useful for both developing piuparts as well as doing QA on the
Debian archive or other .deb archives.
See README_server.txt and README_piatti.txt in the piuparts-master
package for more information.
-- Holger Levsen Mon, 18 Jun 2012 09:26:27 +0200
piuparts (0.38) unstable; urgency=low
New default behaviours in piuparts:
* --skip-minimize is now the default. This is because debootstrap is called
with --variant=minbase instead of --resolv-deps now and because if a
base.tgz or an lvm snapshot is specified, piuparts should not touch it.
(Closes: #539142)
Introduce a new option, --minimize, to minimize a chroot with debfoster.
(This is the old default behaviour. In future versions the --minimize
option might be removed again.)
Add two new options: --keyring to specify the keyring to use with
debootstrap (defaults to /usr/share/keyrings/debian-archive-keyring.gpg)
and --do-not-verify-signatures to not use --keyring with debootstrap.
(Closes: #545907)
In the chroots, APT::Get::AllowUnauthenticated is set accordingly in
/etc/apt/apt.conf.
Add another type of custom-scripts, post_setup_, which are executed after
the chroot was setup. (Closes: #466043)
-- Holger Levsen Thu, 17 Dec 2009 14:10:27 +0000
piuparts (0.36) unstable; urgency=low
piuparts 0.36 introduces many changes to master-slave mode, please read
debian/changelog carefully.
Backwards compatibility of the configuration files for master-slave-mode
has been broken, as the three config files got merged into one:
/etc/piuparts/piuparts.conf
piuparts-reports has been re-added and improved.
The documentation files have been merged and converted to asciidoc.
-- Holger Levsen Tue, 10 Mar 2009 15:23:59 +0100
piuparts-0.56/debian/copyright 0000664 0000000 0000000 00000004327 12144653075 013375 0 ustar Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: piuparts
Upstream-Contact: Holger Levsen
Source: http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git
Copyright: 2005-2008 Lars Wirzenius
2008-2011 Holger Levsen
© 2011-2013 Andreas Beckmann
Comment: Original author is also Lars Wirzenius
Upstream is bunch of lunatics who don't make release tarballs publicly
available. The Debian .orig.tar.gz is what would be released, though.
Files: *
Copyright: 2005-2008 Lars Wirzenius
2008-2011 Holger Levsen
© 2011-2013 Andreas Beckmann
License: GPL-2+
This program is free software; you can redistribute it
and/or modify it under the terms of the GNU General Public
License as published by the Free Software Foundation; either
version 2 of the License, or (at your option) any later
version.
.
This program is distributed in the hope that it will be
useful, but WITHOUT ANY WARRANTY; without even the implied
warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the GNU General Public License for more
details.
.
You should have received a copy of the GNU General Public
License along with this package; if not, write to the Free
Software Foundation, Inc., 51 Franklin St, Fifth Floor,
Boston, MA 02110-1301 USA
.
On Debian systems, the full text of the GNU General Public
License version 2 can be found in the file
'/usr/share/common-licenses/GPL-2'.
Files: */debian.png */openlogo-ng-50.png
Copyright: 1999 Software in the Public Interest
License: Free
This logo or a modified version may be used by anyone to refer to the Debian project, but does not indicate endorsement by the project.
.
Note: we would appreciate that you make the image a link to http://www.debian.org/ if you use it on a web page.
Files: */valid-html401.png */w3c-valid-css.png
Copyright: 2009 W3C
License: Free Use
In most cases, W3C logos and icons may be used without requesting permission
from W3C. When the logo usage policy requires that you obtain express written
permission, please send your request to the W3C Communications Team
(w3t-pr@w3.org)
piuparts-0.56/debian/piuparts.manpages 0000664 0000000 0000000 00000000055 12144650665 015022 0 ustar debian/tmp/usr/share/man/man1/piuparts.1.gz
piuparts-0.56/debian/piuparts.postrm 0000664 0000000 0000000 00000000235 12144650665 014553 0 ustar #!/bin/sh
set -e
if [ "$1" = "purge" ] ; then
# may be left over from a previous version
rmdir /etc/piuparts 2>/dev/null || true
fi
#DEBHELPER#
piuparts-0.56/debian/piuparts.install 0000664 0000000 0000000 00000000052 12144650665 014672 0 ustar /usr/sbin/piuparts
/etc/piuparts/scripts*
piuparts-0.56/debian/piuparts-slave.postinst 0000664 0000000 0000000 00000003753 12144653075 016230 0 ustar #!/bin/sh
set -e
pgroup=piuparts
muser=piupartsm
suser=piupartss
scriptroot=/usr/share/piuparts
userhome=/var/lib/piuparts
if [ "$1" = "configure" ] ; then
addgroup --system --quiet $pgroup
for user in $suser
do
adduser --system --quiet --home $userhome/$user --ingroup $pgroup \
--shell /bin/sh $user
if ! [ -d $userhome/$user ]; then
mkdir -m 0755 $userhome/$user
chown $user:$pgroup $userhome/$user
fi
touch $userhome/$user/.profile
chown $user:$pgroup $userhome/$user/.profile
if [ $user = "piupartsm" ] ;
then
role="master"
else
role="slave"
fi
if ! grep -q $scriptroot/$role $userhome/$user/.profile ;
then
echo PATH=\$PATH:$scriptroot:$scriptroot/$role \
>>$userhome/$user/.profile
fi
if [ ! -d $userhome/$user/.ssh ] ; then
install -d -o $user -g $pgroup -m 0755 $userhome/$user/.ssh
fi
done
if [ ! -e $userhome/$suser/.ssh/id_rsa ] ; then
ssh-keygen -q -f $userhome/$suser/.ssh/id_rsa -N ""
chown $suser:$pgroup $userhome/$suser/.ssh/id_rsa \
$userhome/$suser/.ssh/id_rsa.pub
fi
if [ ! -e $userhome/$suser/.ssh/known_hosts ] ; then
ssh-keyscan localhost > $userhome/$suser/.ssh/known_hosts 2>/dev/null
chown $suser:$pgroup $userhome/$suser/.ssh/known_hosts
fi
if [ -d $userhome/$muser/.ssh ] && [ ! -e $userhome/$muser/.ssh/authorized_keys ] ; then
sed 's#^#command="'"$scriptroot"'/piuparts-master",no-pty,no-port-forwarding #' \
$userhome/$suser/.ssh/id_rsa.pub >>$userhome/$muser/.ssh/authorized_keys
chown $muser:$pgroup $userhome/$muser/.ssh/authorized_keys
chmod 0600 $userhome/$muser/.ssh/authorized_keys
fi
install -d -o $suser -g $pgroup -m 0755 $userhome/slave
install -d -o $suser -g $pgroup -m 0755 $userhome/slave/0
chown $suser:$pgroup /var/cache/piuparts/basetgz
fi
#DEBHELPER#
piuparts-0.56/debian/piuparts-slave.manpages 0000664 0000000 0000000 00000000155 12155061511 016117 0 ustar debian/tmp/usr/share/man/man8/piuparts_slave_run.8.gz
debian/tmp/usr/share/man/man8/piuparts_slave_join.8.gz
piuparts-0.56/debian/piuparts.docs 0000664 0000000 0000000 00000000077 12155061511 014147 0 ustar README_1st.txt
README_1st.html
piuparts.1.html
docbook-xsl.css
piuparts-0.56/debian/piuparts-master.docs 0000664 0000000 0000000 00000000101 12151411561 015424 0 ustar README_server.txt
README_pejacevic.txt
conf/piuparts.conf.sample
piuparts-0.56/debian/piuparts-master.postrm 0000664 0000000 0000000 00000000755 12144650665 016053 0 ustar #!/bin/sh
set -e
user=piupartsm
if [ "$1" = "purge" ] ; then
rm -rf /var/lib/piuparts/$user
if [ -d /var/lib/piuparts/htdocs ] ; then
rm -rf /var/lib/piuparts/htdocs
fi
if [ -d /var/lib/piuparts/backup ] ; then
rm -rf /var/lib/piuparts/backup
fi
if [ -d /var/lib/piuparts/master ] ; then
rm -rf /var/lib/piuparts/master
fi
rmdir /var/lib/piuparts 2>/dev/null || true
rmdir /etc/piuparts 2>/dev/null || true
fi
#DEBHELPER#
piuparts-0.56/debian/control 0000664 0000000 0000000 00000006672 12323207446 013046 0 ustar Source: piuparts
Maintainer: Ubuntu Developers
XSBC-Original-Maintainer: piuparts developers team
Uploaders:
Holger Levsen
Section: devel
Priority: extra
Standards-Version: 3.9.5
Build-Depends:
debhelper (>= 9.20120909~),
python (>= 2.7),
Build-Depends-Indep:
asciidoc,
git,
xmlto
Homepage: http://piuparts.debian.org
Vcs-Git: git://anonscm.debian.org/piuparts/piuparts.git
Vcs-Browser: http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git
X-Python-Version: >= 2.7
Package: piuparts
Architecture: all
Pre-Depends:
dpkg (>= 1.15.7.2~),
Depends:
piuparts-common (= ${binary:Version}),
debootstrap,
debsums,
lsb-release,
lsof,
python-debian,
${misc:Depends},
${python:Depends}
Recommends:
adequate
Suggests:
schroot
Description: .deb package installation, upgrading, and removal testing tool
piuparts tests that .deb packages (as used by Debian) handle
installation, upgrading, and removal correctly. It does this by
creating a minimal Debian installation in a chroot, and installing,
upgrading, and removing packages in that environment, and comparing the
state of the directory tree before and after. piuparts reports any
files that have been added, removed, or modified during this process.
.
piuparts is meant as a quality assurance tool for people who create .deb
packages to test them before they upload them to the Debian package archive.
Package: piuparts-master
Architecture: all
Depends:
piuparts-common (= ${binary:Version}),
adduser,
openssh-server,
python-debianbts,
tango-icon-theme,
xz-utils,
python-yaml,
${misc:Depends},
${python:Depends}
Recommends:
apache2 | httpd,
ghostscript,
python-rpy2,
r-base-dev,
r-recommended,
devscripts,
libsoap-lite-perl,
Breaks:
piuparts (<= 0.44)
Replaces:
piuparts (<= 0.44)
Description: piuparts master components
piuparts is meant as a quality assurance tool for people who create .deb
packages to test them before they upload them to the Debian package archive.
A master/slave piuparts installation is used for testing complete archives
as it is done on http://piuparts.debian.org/.
.
This package provides the piuparts-master, which needs one or more slaves to
operate. It also contains piuparts-reports to generate web pages with the
testing results as well as documentation.
Package: piuparts-slave
Architecture: all
Depends:
piuparts (= ${binary:Version}),
piuparts-common (= ${binary:Version}),
adduser,
openssh-client,
screen,
sudo,
${misc:Depends},
${python:Depends}
Suggests:
schroot
Breaks:
piuparts (<= 0.44)
Replaces:
piuparts (<= 0.44)
Description: piuparts slave components
piuparts is meant as a quality assurance tool for people who create .deb
packages to test them before they upload them to the Debian package archive.
A master/slave piuparts installation is used for testing complete archives
as it is done on http://piuparts.debian.org/.
.
This package provides the piuparts slave components, which need a piuparts
master to operate. See the documentation in the piuparts-master package.
Package: piuparts-common
Architecture: all
Depends:
python-apt,
${misc:Depends},
${python:Depends}
Breaks:
piuparts (<= 0.44)
Replaces:
piuparts (<= 0.44)
Description: common piuparts components
piupartslib, common library used by piuparts-master, -reports, -analyze and
-slave.
.
See http://piuparts.debian.org or the piuparts package to learn more about
piuparts.
piuparts-0.56/debian/compat 0000664 0000000 0000000 00000000002 12253565512 012631 0 ustar 9
piuparts-0.56/debian/piuparts-slave.install 0000664 0000000 0000000 00000000211 12144650665 015777 0 ustar etc/cron.d/piuparts-slave
etc/piuparts/piuparts.conf
etc/sudoers.d/piuparts
usr/share/piuparts/piuparts-slave
usr/share/piuparts/slave/*
piuparts-0.56/debian/rules 0000775 0000000 0000000 00000001063 12151254042 012501 0 ustar #!/usr/bin/make -f
# Uncomment this to turn on verbose mode.
#export DH_VERBOSE=1
%:
dh $@ --with python2
override_dh_auto_build:
$(MAKE) prefix=/usr build build-doc
override_dh_auto_test:
echo "unittests are disabled as they haven't been run at build time since years and thus are broken..."
override_dh_auto_install:
$(MAKE) DESTDIR=$(CURDIR)/debian/tmp prefix=/usr etcdir=/etc install install-doc install-conf
override_dh_python2:
dh_python2 -p piuparts-master -p piuparts-slave /usr/share/piuparts
dh_python2 -N piuparts-master -N piuparts-slave
piuparts-0.56/debian/source/ 0000775 0000000 0000000 00000000000 12144650665 012736 5 ustar piuparts-0.56/debian/source/format 0000664 0000000 0000000 00000000004 12144650665 014143 0 ustar 1.0
piuparts-0.56/debian/source/options 0000664 0000000 0000000 00000000022 12144650665 014346 0 ustar tar-ignore = .git
piuparts-0.56/debian/piuparts-slave.dirs 0000664 0000000 0000000 00000000103 12144650665 015272 0 ustar var/lib/piuparts
var/cache/piuparts/basetgz
var/cache/piuparts/tmp
piuparts-0.56/debian/piuparts-master.dirs 0000664 0000000 0000000 00000000021 12144650665 015452 0 ustar var/lib/piuparts
piuparts-0.56/debian/piuparts-master.postinst 0000664 0000000 0000000 00000003253 12144650665 016406 0 ustar #!/bin/sh
set -e
pgroup=piuparts
muser=piupartsm
scriptroot=/usr/share/piuparts
userhome=/var/lib/piuparts
if [ "$1" = "configure" ] ; then
addgroup --system --quiet $pgroup
for user in $muser
do
adduser --system --quiet --home $userhome/$user --ingroup $pgroup \
--shell /bin/sh $user
if ! [ -d $userhome/$user ]; then
mkdir -m 0755 $userhome/$user
chown $user:$pgroup $userhome/$user
fi
touch $userhome/$user/.profile
chown $user:$pgroup $userhome/$user/.profile
if [ $user = "piupartsm" ] ;
then
role="master"
else
role="slave"
fi
if ! grep -q $scriptroot/$role $userhome/$user/.profile ;
then
echo PATH=\$PATH:$scriptroot:$scriptroot/$role \
>>$userhome/$user/.profile
fi
if [ ! -d $userhome/$user/.ssh ] ; then
install -d -o $user -g $pgroup -m 0755 $userhome/$user/.ssh
fi
done
install -d -o $muser -g $pgroup -m 0755 $userhome/master
install -d -o $muser -g $pgroup -m 0755 $userhome/backup
install -d -o $muser -g $pgroup -m 0755 $userhome/htdocs
cp -rf $scriptroot/htdocs/* $userhome/htdocs/
for icon in weather-severe-alert.png sunny.png
do
if [ -h $userhome/htdocs/images/$icon ] ; then
rm -f $userhome/htdocs/images/$icon
fi
if [ -e /usr/share/icons/Tango/24x24/status/$icon ] ; then
cp -f /usr/share/icons/Tango/24x24/status/$icon $userhome/htdocs/images/$icon
fi
done
if dpkg --compare-versions "$2" lt-nl "0.50" ; then
rm -rf $userhome/master/monitor-slave
fi
fi
#DEBHELPER#
piuparts-0.56/debian/piuparts.maintscript 0000664 0000000 0000000 00000000377 12144650665 015573 0 ustar rm_conffile /etc/piuparts/piuparts.conf 0.45~
rm_conffile /etc/piuparts/scripts-leftovers/post_purge_dbconfig_cleanup 0.47~
mv_conffile /etc/piuparts/scripts/post_install_find_bad_permissions /etc/piuparts/scripts/pre_remove_50_find_bad_permissions 0.47~
piuparts-0.56/debian/piuparts-slave.postrm 0000664 0000000 0000000 00000001120 12151074163 015644 0 ustar #!/bin/sh
set -e
user=piupartss
if [ "$1" = "purge" ] ; then
rm -rf /var/lib/piuparts/$user
if [ -d /var/lib/piuparts/slave ] ; then
rm -rf /var/lib/piuparts/slave
fi
rmdir /var/lib/piuparts 2>/dev/null || true
rmdir /etc/piuparts 2>/dev/null || true
rm -rf /var/cache/piuparts/basetgz/*.log
rm -rf /var/cache/piuparts/basetgz/*.tar.gz
# rm -rf is safe if it's not mounted anymore
mount | grep -q /var/cache/piuparts/tmp || rm -rf --one-file-system /var/cache/piuparts/tmp
rmdir /var/cache/piuparts 2>/dev/null || true
fi
#DEBHELPER#
piuparts-0.56/debian/piuparts-master.install 0000664 0000000 0000000 00000000515 12144653075 016165 0 ustar etc/apache2/conf.d/piuparts.apache
etc/cron.d/piuparts-master
#etc/piuparts/known_problems/
usr/share/piuparts/htdocs/*
usr/share/piuparts/master/*
usr/share/piuparts/known_problems/*
usr/share/piuparts/piuparts-analyze
usr/share/piuparts/piuparts-master
usr/share/piuparts/piuparts-master-backend
usr/share/piuparts/piuparts-report
piuparts-0.56/lib/ 0000775 0000000 0000000 00000000000 12144653075 010760 5 ustar piuparts-0.56/lib/read_config.sh 0000664 0000000 0000000 00000003760 12144653075 013562 0 ustar # Copyright © 2011, 2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Helper function for getting values from piuparts.conf.
# Used by several master and slave scripts.
#
PIUPARTS_CONF=${PIUPARTS_CONF:-/etc/piuparts/piuparts.conf}
[ -f "$PIUPARTS_CONF" ] || exit 0
# usage: get_config_value VARIABLE section key [default]
get_config_value()
{
local section key value
test -n "$1" && test "$1" = "$(echo "$1" | tr -c -d '[:alnum:]_')" || exit 1
section="$2"
key="$3"
# First select the [$section] block (\#^\[$section\]#) (use # as
# marker because $section may contain slashes) up to the start of the
# next section (/^\[/). The select the $key=value, this may be wrapped
# with indented lines and comment lines embedded. The $key=value is
# over once we hit the next key (or any line not starting with # or
# whitespace. Throw away comments (/^#/d), the following key, remove
# our $key= part, trim the value, remove empty lines, and print it.
value="$(sed -rn '\#^\['"$section"'\]#,/^\[/ {/^'"$key"'\s*=/,/^[^ \t#]/ {/^#/d; /^'"$key"'\s*=|^\s/!d; s/^'"$key"'\s*=\s*//; s/^\s*//; s/\s*$//; /^$/d; p}}' "$PIUPARTS_CONF")"
if [ -z "$value" ]; then
if [ -n "${4+set}" ]; then
value="$4"
else
echo "'$key' not set in section [$section] of $PIUPARTS_CONF, exiting." >&2
exit 1
fi
fi
eval "$1"='"$value"'
}
piuparts-0.56/piuparts-slave.py 0000664 0000000 0000000 00000103755 12253565512 013555 0 ustar #!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2005 Lars Wirzenius (liw@iki.fi)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
"""Distributed piuparts processing, slave program
Lars Wirzenius
"""
import os
import sys
import stat
import time
import logging
from signal import alarm, signal, SIGALRM, SIGINT, SIGKILL, SIGHUP
import subprocess
import fcntl
import random
import ConfigParser
import apt_pkg
import piupartslib.conf
import piupartslib.packagesdb
from piupartslib.conf import MissingSection
apt_pkg.init_system()
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
MAX_WAIT_TEST_RUN = 45*60
interrupted = False
old_sigint_handler = None
got_sighup = False
def setup_logging(log_level, log_file_name):
logger = logging.getLogger()
logger.setLevel(log_level)
formatter = logging.Formatter(fmt="%(asctime)s %(message)s",
datefmt="%H:%M:%S")
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(formatter)
logger.addHandler(handler)
if log_file_name:
handler = logging.FileHandler(log_file_name)
logger.addHandler(handler)
class Config(piupartslib.conf.Config):
def __init__(self, section="slave", defaults_section=None):
self.section = section
piupartslib.conf.Config.__init__(self, section,
{
"sections": "slave",
"idle-sleep": 300,
"max-tgz-age": 2592000,
"min-tgz-retry-delay": 21600,
"master-host": None,
"master-user": None,
"master-command": None,
"proxy": None,
"mirror": None,
"piuparts-command": "sudo piuparts",
"piuparts-flags": "",
"tmpdir": None,
"distro": None,
"area": None,
"components": None,
"chroot-tgz": None,
"upgrade-test-distros": None,
"basetgz-directory": ".",
"max-reserved": 1,
"debug": "no",
"keep-sources-list": "no",
"arch": None,
"precedence": "1",
"slave-load-max": None,
},
defaults_section=defaults_section)
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
def sigint_handler(signum, frame):
global interrupted
interrupted = True
print '\nSlave interrupted by the user, waiting for the current test to finish.'
print 'Press Ctrl-C again to abort now.'
signal(SIGINT, old_sigint_handler)
def sighup_handler(signum, frame):
global got_sighup
got_sighup = True
print 'SIGHUP: Will flush finished logs.'
class MasterIsBusy(Exception):
def __init__(self):
self.args = "Master is busy, retry later",
class MasterNotOK(Exception):
def __init__(self):
self.args = "Master did not respond with 'ok'",
class MasterDidNotGreet(Exception):
def __init__(self):
self.args = "Master did not start with 'hello'",
class MasterCommunicationFailed(Exception):
def __init__(self):
self.args = "Communication with master failed",
class MasterIsCrazy(Exception):
def __init__(self):
self.args = "Master said something unexpected",
class MasterCantRecycle(Exception):
def __init__(self):
self.args = "Master has nothing to recycle",
class Slave:
def __init__(self):
self._to_master = None
self._from_master = None
self._master_host = None
self._master_user = None
self._master_command = None
self._section = None
def _readline(self):
try:
line = self._from_master.readline()
except IOError:
raise MasterCommunicationFailed()
logging.debug("<< " + line.rstrip())
return line
def _writeline(self, *words):
line = " ".join(words)
logging.debug(">> " + line)
try:
self._to_master.write(line + "\n")
self._to_master.flush()
except IOError:
raise MasterCommunicationFailed()
def set_master_host(self, host):
logging.debug("Setting master host to %s" % host)
if self._master_host != host:
self.close()
self._master_host = host
def set_master_user(self, user):
logging.debug("Setting master user to %s" % user)
if self._master_user != user:
self.close()
self._master_user = user
def set_master_command(self, cmd):
logging.debug("Setting master command to %s" % cmd)
if self._master_command != cmd:
self.close()
self._master_command = cmd
def set_section(self, section):
logging.debug("Setting section to %s" % section)
self._section = section
def connect_to_master(self):
if not self._is_connected():
self._initial_connect()
self._select_section()
def _is_connected(self):
return self._to_master and self._from_master
def _initial_connect(self):
logging.info("Connecting to %s" % self._master_host)
ssh_command = ["ssh", "-x"]
if self._master_user:
ssh_command.extend(["-l", self._master_user])
ssh_command.append(self._master_host)
ssh_command.append(self._master_command or "command-is-set-in-authorized_keys")
p = subprocess.Popen(ssh_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
self._to_master = p.stdin
self._from_master = p.stdout
line = self._readline()
if line != "hello\n":
raise MasterDidNotGreet()
def _select_section(self):
self._writeline("section", self._section)
line = self._readline()
if line == "busy\n":
raise MasterIsBusy()
elif line != "ok\n":
raise MasterNotOK()
logging.debug("Connected to master")
def close(self):
if self._from_master is None and self._to_master is None:
return
logging.debug("Closing connection to master")
if self._from_master is not None:
self._from_master.close()
if self._to_master is not None:
self._to_master.close()
self._from_master = self._to_master = None
logging.info("Connection to master closed")
def send_log(self, section, pass_or_fail, filename):
logging.info("Sending log file %s/%s" % (section, filename))
basename = os.path.basename(filename)
package, rest = basename.split("_", 1)
version = rest[:-len(".log")]
self._writeline(pass_or_fail, package, version)
f = file(filename, "r")
for line in f:
if line.endswith("\n"):
line = line[:-1]
self._writeline(" " + line)
f.close()
self._writeline(".")
line = self._readline()
if line != "ok\n":
raise MasterNotOK()
def get_status(self, section):
self._writeline("status")
line = self._readline()
words = line.split()
if words and words[0] == "ok":
logging.info("Master " + section + " status: " + " ".join(words[1:]))
else:
raise MasterIsCrazy()
def enable_recycling(self):
self._writeline("recycle")
line = self._readline()
words = line.split()
if line != "ok\n":
raise MasterCantRecycle()
def get_idle(self):
self._writeline("idle")
line = self._readline()
words = line.split()
if words and words[0] == "ok" and len(words) == 2:
return int(words[1])
else:
raise MasterIsCrazy()
def reserve(self):
self._writeline("reserve")
line = self._readline()
words = line.split()
if words and words[0] == "ok":
logging.info("Reserved for us: %s %s" % (words[1], words[2]))
self.remember_reservation(words[1], words[2])
return True
elif words and words[0] == "error":
logging.info("Master didn't reserve anything (more) for us")
return False
else:
raise MasterIsCrazy()
def unreserve(self, filename):
basename = os.path.basename(filename)
package, rest = basename.split("_", 1)
version = rest[:-len(".log")]
logging.info("Unreserve: %s %s" % (package, version))
self._writeline("unreserve", package, version)
line = self._readline()
if line != "ok\n":
raise MasterNotOK()
def _reserved_filename(self, name, version):
return os.path.join("reserved", "%s_%s.log" % (name, version))
def remember_reservation(self, name, version):
create_file(self._reserved_filename(name, version), "")
def get_reserved(self):
vlist = []
for basename in os.listdir("reserved"):
if "_" in basename and basename.endswith(".log"):
name, version = basename[:-len(".log")].split("_", 1)
vlist.append((name, version))
return vlist
def forget_reserved(self, name, version):
try:
os.remove(self._reserved_filename(name, version))
except os.error:
pass
class Section:
def __init__(self, section, slave=None):
self._config = Config(section=section, defaults_section="global")
self._config.read(CONFIG_FILE)
self._distro_config = piupartslib.conf.DistroConfig(
DISTRO_CONFIG_FILE, self._config["mirror"])
self._error_wait_until = 0
self._idle_wait_until = 0
self._recycle_wait_until = 0
self._slave_directory = os.path.abspath(section)
if not os.path.exists(self._slave_directory):
os.makedirs(self._slave_directory)
if self._config["debug"] in ["yes", "true"]:
self._logger = logging.getLogger()
self._logger.setLevel(logging.DEBUG)
if int(self._config["max-reserved"]) > 0:
self._check_tarball()
for rdir in ["new", "pass", "fail", "untestable", "reserved"]:
rdir = os.path.join(self._slave_directory, rdir)
if not os.path.exists(rdir):
os.mkdir(rdir)
self._slave = slave or Slave()
def _throttle_if_overloaded(self):
global interrupted
if interrupted or got_sighup:
return
if self._config["slave-load-max"] is None:
return
load_max = float(self._config["slave-load-max"])
if load_max < 1.0:
return
if os.getloadavg()[0] <= load_max:
return
load_resume = max(load_max - 1.0, 0.9)
secs = random.randrange(30, 90)
self._slave.close()
while True:
load = os.getloadavg()[0]
if load <= load_resume:
break
logging.info("Sleeping due to high load (%.2f)" % load)
try:
time.sleep(secs)
except KeyboardInterrupt:
interrupted = True
if interrupted or got_sighup:
break
if secs < 300:
secs += random.randrange(30, 90)
def _connect_to_master(self, recycle=False):
self._slave.set_master_host(self._config["master-host"])
self._slave.set_master_user(self._config["master-user"])
self._slave.set_master_command(self._config["master-command"])
self._slave.set_section(self._config.section)
self._slave.connect_to_master()
if recycle:
self._slave.enable_recycling()
def _get_tarball(self):
basetgz = self._config["chroot-tgz"] or \
self._distro_config.get_basetgz(self._config.get_start_distro(),
self._config.get_arch())
return os.path.join(self._config["basetgz-directory"], basetgz)
def _check_tarball(self):
oldcwd = os.getcwd()
os.chdir(self._slave_directory)
tgz = self._get_tarball()
max_tgz_age = int(self._config["max-tgz-age"])
min_tgz_retry_delay = int(self._config["min-tgz-retry-delay"])
needs_update = not os.path.exists(tgz)
if not needs_update and max_tgz_age > 0:
# tgz exists and age is limited, so check age
now = time.time()
age = now - os.path.getmtime(tgz)
logging.info("Check-replace %s: age=%d vs. max=%d" % (tgz, age, max_tgz_age))
if age > max_tgz_age:
if os.path.exists(tgz + ".log"):
age = now - os.path.getmtime(tgz + ".log")
logging.info("Limit-replace %s: last-retry=%d vs. min=%d" % (tgz, age, min_tgz_retry_delay))
if age > min_tgz_retry_delay:
needs_update = True
logging.info("%s too old. Forcing re-creation" % tgz)
if needs_update:
create_chroot(self._config, tgz, self._config.get_start_distro())
os.chdir(oldcwd)
def _count_submittable_logs(self):
files = 0
subdirs = ["pass", "fail", "untestable"]
if interrupted:
subdirs += ["reserved", "new"]
for logdir in subdirs:
for basename in os.listdir(os.path.join(self._slave_directory, logdir)):
if basename.endswith(".log"):
files += 1
return files
def precedence(self):
return int(self._config["precedence"])
def sleep_until(self, recycle=False):
if recycle:
return max(self._error_wait_until, self._recycle_wait_until)
return max(self._error_wait_until, self._idle_wait_until)
def run(self, do_processing=True, recycle=False):
if time.time() < self.sleep_until(recycle=recycle):
return 0
self._throttle_if_overloaded()
if interrupted or got_sighup:
do_processing = False
if not do_processing and self._count_submittable_logs() == 0:
return 0
logging.info("-------------------------------------------")
action = "Running"
if recycle:
action = "Recycling"
if not do_processing:
action = "Flushing"
logging.info("%s section %s (precedence=%d)" \
% (action, self._config.section, self.precedence()))
self._config = Config(section=self._config.section, defaults_section="global")
try:
self._config.read(CONFIG_FILE)
except MissingSection:
logging.info("unknown")
self._error_wait_until = time.time() + 3600
return 0
self._distro_config = piupartslib.conf.DistroConfig(
DISTRO_CONFIG_FILE, self._config["mirror"])
if int(self._config["max-reserved"]) == 0:
logging.info("disabled")
self._error_wait_until = time.time() + 12 * 3600
return 0
if not self._config.get_distro() and not self._config.get_distros():
logging.error("neither 'distro' nor 'upgrade-test-distros' configured")
self._error_wait_until = time.time() + 3600
return 0
with open(os.path.join(self._slave_directory, "slave.lock"), "we") as lock:
oldcwd = os.getcwd()
os.chdir(self._slave_directory)
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
logging.info("busy")
self._error_wait_until = time.time() + 900
else:
if self._talk_to_master(fetch=do_processing, recycle=recycle, unreserve=interrupted):
if do_processing:
if not self._slave.get_reserved():
self._idle_wait_until = time.time() + int(self._config["idle-sleep"])
if recycle:
self._recycle_wait_until = self._idle_wait_until + 3600
else:
processed = self._process()
if got_sighup and self._slave.get_reserved():
# keep this section at the front of the round-robin runnable queue
self._idle_wait_until = 0
self._recycle_wait_until = 0
else:
# put this section at the end of the round-robin runnable queue
self._idle_wait_until = time.time()
self._recycle_wait_until = time.time()
return processed
finally:
os.chdir(oldcwd)
return 0
def _talk_to_master(self, fetch=False, unreserve=False, recycle=False):
flush = self._count_submittable_logs() > 0
fetch = fetch and not self._slave.get_reserved()
if not flush and not fetch:
return True
try:
self._connect_to_master(recycle=recycle)
except KeyboardInterrupt:
raise
except MasterIsBusy:
logging.error("master is busy")
self._error_wait_until = time.time() + random.randrange(60, 180)
except MasterCantRecycle:
logging.error("master has nothing to recycle")
self._recycle_wait_until = max(time.time(), self._idle_wait_until) + 3600
except (MasterDidNotGreet, MasterIsCrazy, MasterCommunicationFailed):
logging.error("connection to master failed")
self._error_wait_until = time.time() + 900
self._slave.close()
else:
try:
for logdir in ["pass", "fail", "untestable"]:
for basename in os.listdir(logdir):
if basename.endswith(".log"):
fullname = os.path.join(logdir, basename)
self._slave.send_log(self._config.section, logdir, fullname)
os.remove(fullname)
if unreserve:
for logdir in ["new", "reserved"]:
for basename in os.listdir(logdir):
if basename.endswith(".log"):
fullname = os.path.join(logdir, basename)
self._slave.unreserve(fullname)
os.remove(fullname)
if fetch:
max_reserved = int(self._config["max-reserved"])
idle = self._slave.get_idle()
if idle > 0:
idle = min(idle, int(self._config["idle-sleep"]))
logging.info("idle (%d)" % idle)
if not recycle:
self._idle_wait_until = time.time() + idle
else:
self._recycle_wait_until = time.time() + idle
return 0
while len(self._slave.get_reserved()) < max_reserved and self._slave.reserve():
pass
self._slave.get_status(self._config.section)
except MasterNotOK:
logging.error("master did not respond with 'ok'")
self._error_wait_until = time.time() + 900
self._slave.close()
except (MasterIsCrazy, MasterCommunicationFailed):
logging.error("communication with master failed")
self._error_wait_until = time.time() + 900
self._slave.close()
else:
return True
return False
def _process(self):
self._slave.close()
packagenames = set([x[0] for x in self._slave.get_reserved()])
packages_files = {}
for distro in [self._config.get_distro()] + self._config.get_distros():
if distro not in packages_files:
try:
pf = piupartslib.packagesdb.PackagesFile()
pf.load_packages_urls(
self._distro_config.get_packages_urls(
distro,
self._config.get_area(),
self._config.get_arch()),
packagenames)
packages_files[distro] = pf
except IOError:
logging.error("failed to fetch packages file for %s" % distro)
self._error_wait_until = time.time() + 900
return 0
del packagenames
test_count = 0
self._check_tarball()
for package_name, version in self._slave.get_reserved():
self._throttle_if_overloaded()
if interrupted or got_sighup:
break
if not os.path.exists(self._get_tarball()):
logging.error("Missing chroot-tgz %s" % self._get_tarball())
break
test_count += 1
self._test_package(package_name, version, packages_files)
self._slave.forget_reserved(package_name, version)
self._talk_to_master(unreserve=interrupted)
return test_count
def _test_package(self, pname, pvers, packages_files):
global old_sigint_handler
old_sigint_handler = signal(SIGINT, sigint_handler)
logging.info("Testing package %s/%s %s" % (self._config.section, pname, pvers))
output_name = log_name(pname, pvers)
logging.debug("Opening log file %s" % output_name)
new_name = os.path.join("new", output_name)
output = file(new_name, "we")
output.write(time.strftime("Start: %Y-%m-%d %H:%M:%S %Z\n",
time.gmtime()))
distupgrade = len(self._config.get_distros()) > 1
command = self._config["piuparts-command"].split()
if self._config["piuparts-flags"]:
command.extend(self._config["piuparts-flags"].split())
if "http_proxy" in os.environ:
command.extend(["--proxy", os.environ["http_proxy"]])
if self._config["mirror"]:
mirror = self._config["mirror"]
if self._config["components"]:
mirror += " " + self._config["components"]
command.extend(["--mirror", mirror])
if self._config["tmpdir"]:
command.extend(["--tmpdir", self._config["tmpdir"]])
command.extend(["--arch", self._config.get_arch()])
command.extend(["-b", self._get_tarball()])
if not distupgrade:
command.extend(["-d", self._config.get_distro()])
command.append("--no-upgrade-test")
else:
for distro in self._config.get_distros():
command.extend(["-d", distro])
if self._config["keep-sources-list"] in ["yes", "true"]:
command.append("--keep-sources-list")
command.extend(["--apt", "%s=%s" % (pname, pvers)])
subdir = "fail"
ret = 0
if not distupgrade:
distro = self._config.get_distro()
if not pname in packages_files[distro]:
output.write("Package %s not found in %s\n" % (pname, distro))
ret = -10001
else:
package = packages_files[distro][pname]
if pvers != package["Version"]:
output.write("Package %s %s not found in %s, %s is available\n" % (pname, pvers, distro, package["Version"]))
ret = -10002
output.write("\n")
package.dump(output)
output.write("\n")
else:
distros = self._config.get_distros()
if distros:
# the package must exist somewhere
for distro in distros:
if pname in packages_files[distro]:
break
else:
output.write("Package %s not found in any distribution\n" % pname)
ret = -10003
# the package must have the correct version in the distupgrade target distro
distro = distros[-1]
if not pname in packages_files[distro]:
# the package may "disappear" in the distupgrade target distro
if pvers == "None":
pass
else:
output.write("Package %s not found in %s\n" % (pname, distro))
ret = -10004
else:
package = packages_files[distro][pname]
if pvers != package["Version"]:
output.write("Package %s %s not found in %s, %s is available\n" % (pname, pvers, distro, package["Version"]))
ret = -10005
for distro in distros:
output.write("\n[%s]\n" % distro)
if pname in packages_files[distro]:
packages_files[distro][pname].dump(output)
output.write("\n")
if ret == 0:
prev = "~"
for distro in distros:
if pname in packages_files[distro]:
v = packages_files[distro][pname]["Version"]
if not apt_pkg.version_compare(prev, v) <= 0:
output.write("Upgrade to %s requires downgrade: %s > %s\n" % (distro, prev, v))
ret = -10006
prev = v
else:
ret = -10010
if ret != 0:
subdir = "untestable"
if ret == 0:
output.write("Executing: %s\n" % " ".join(quote_spaces(command)))
ret, f = run_test_with_timeout(command, MAX_WAIT_TEST_RUN)
if not f or f[-1] != '\n':
f += '\n'
output.write(f)
lastline = f.split('\n')[-2]
if ret < 0:
output.write(" *** Process KILLED - exceed maximum run time ***\n")
elif not "piuparts run ends" in lastline:
ret += 1024
output.write(" *** PIUPARTS OUTPUT INCOMPLETE ***\n");
output.write("\n")
output.write("ret=%d\n" % ret)
output.write(time.strftime("End: %Y-%m-%d %H:%M:%S %Z\n",
time.gmtime()))
output.close()
if ret == 0:
subdir = "pass"
os.rename(new_name, os.path.join(subdir, output_name))
logging.debug("Done with %s: %s (%d)" % (output_name, subdir, ret))
signal(SIGINT, old_sigint_handler)
def log_name(package, version):
return "%s_%s.log" % (package, version)
def quote_spaces(vlist):
return ["'%s'" % x if ' ' in x else x for x in vlist]
def run_test_with_timeout(cmd, maxwait, kill_all=True):
def terminate_subprocess(p, kill_all):
pids = [p.pid]
if kill_all:
ps = subprocess.Popen(["ps", "--no-headers", "-o", "pid", "--ppid", "%d" % p.pid],
stdout = subprocess.PIPE)
stdout, stderr = ps.communicate()
pids.extend([int(pid) for pid in stdout.split()])
if p.poll() is None:
print 'Sending SIGINT...'
try:
os.killpg(os.getpgid(p.pid), SIGINT)
except OSError:
pass
# piuparts has 30 seconds to clean up after Ctrl-C
for i in range(60):
time.sleep(0.5)
if p.poll() is not None:
break
if p.poll() is None:
print 'Sending SIGTERM...'
p.terminate()
# piuparts has 5 seconds to clean up after SIGTERM
for i in range(10):
time.sleep(0.5)
if p.poll() is not None:
break
if p.poll() is None:
print 'Sending SIGKILL...'
p.kill()
for pid in pids:
if pid > 0:
try:
os.kill(pid, SIGKILL)
print "Killed %d" % pid
except OSError:
pass
logging.debug("Executing: %s" % " ".join(quote_spaces(cmd)))
stdout = ""
p = subprocess.Popen(cmd, preexec_fn=os.setpgrp,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if maxwait > 0:
signal(SIGALRM, alarm_handler)
alarm(maxwait)
try:
stdout, stderr = p.communicate()
alarm(0)
except Alarm:
terminate_subprocess(p, kill_all)
return -1, stdout
except KeyboardInterrupt:
print '\nSlave interrupted by the user, cleaning up...'
try:
terminate_subprocess(p, kill_all)
except KeyboardInterrupt:
print '\nTerminating piuparts was interrupted... manual cleanup still neccessary.'
raise
raise
ret = p.returncode
if ret in [124, 137]:
# process was terminated by the timeout command
ret = -ret
return ret, stdout
def create_chroot(config, tarball, distro):
command = config["piuparts-command"].split()
if config["piuparts-flags"]:
command.extend(config["piuparts-flags"].split())
if "http_proxy" in os.environ:
command.extend(["--proxy", os.environ["http_proxy"]])
if config["mirror"]:
mirror = config["mirror"]
if config["components"]:
mirror += " " + config["components"]
command.extend(["--mirror", mirror])
if config["tmpdir"]:
command.extend(["--tmpdir", config["tmpdir"]])
command.extend(["--arch", config.get_arch()])
command.extend(["-d", distro])
command.extend(["-s", tarball + ".new"])
command.extend(["--apt", "dpkg"])
output_name = tarball + ".log"
with open(output_name, "we") as output:
try:
fcntl.flock(output, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
logging.info("Creation of tarball %s already in progress." % tarball)
else:
logging.info("Creating new tarball %s" % tarball)
output.write(time.strftime("Start: %Y-%m-%d %H:%M:%S %Z\n\n",
time.gmtime()))
output.write("Executing: " + " ".join(quote_spaces(command)) + "\n\n")
logging.debug("Executing: " + " ".join(quote_spaces(command)))
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
for line in p.stdout:
output.write(line)
logging.debug(">> " + line.rstrip())
p.wait()
output.write(time.strftime("\nEnd: %Y-%m-%d %H:%M:%S %Z\n",
time.gmtime()))
if os.path.exists(tarball + ".new"):
os.rename(tarball + ".new", tarball)
else:
logging.error("Tarball creation failed, see %s" % output_name)
def create_file(filename, contents):
f = file(filename, "w")
f.write(contents)
f.close()
def main():
setup_logging(logging.INFO, None)
signal(SIGHUP, sighup_handler)
# For supporting multiple architectures and suites, we take command-line
# argument(s) referring to section(s) in the configuration file.
# If no argument is given, the "sections" entry from the "global" section
# is used.
section_names = []
global_config = Config(section="global")
global_config.read(CONFIG_FILE)
if global_config["proxy"]:
os.environ["http_proxy"] = global_config["proxy"]
if len(sys.argv) > 1:
section_names = sys.argv[1:]
else:
section_names = global_config["sections"].split()
persistent_connection = Slave()
sections = []
for section_name in section_names:
try:
sections.append(Section(section_name, persistent_connection))
except MissingSection:
# ignore unknown sections
pass
if not sections:
logging.error("no sections found")
return
while True:
global got_sighup
test_count = 0
for section in sorted(sections, key=lambda section: (section.precedence(), section.sleep_until())):
test_count += section.run(do_processing=(test_count == 0))
if test_count == 0 and got_sighup:
# clear SIGHUP state after flushing all sections
got_sighup = False
continue
if test_count == 0:
# try to recycle old logs
# round robin recycling of all sections is ensured by the recycle_wait_until timestamps
idle_until = min([section.sleep_until() for section in sections])
for section in sorted(sections, key=lambda section: section.sleep_until(recycle=True)):
test_count += section.run(recycle=True)
if test_count > 0 and idle_until < time.time():
break
if interrupted:
raise KeyboardInterrupt
if test_count == 0 and not got_sighup:
now = time.time()
sleep_until = min([now + int(global_config["idle-sleep"])] + [section.sleep_until() for section in sections])
if (sleep_until > now):
to_sleep = max(60, sleep_until - now)
persistent_connection.close()
logging.info("Nothing to do, sleeping for %d seconds." % to_sleep)
time.sleep(to_sleep)
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print ''
print 'Slave interrupted by the user, exiting...'
sys.exit(1)
# vi:set et ts=4 sw=4 :
piuparts-0.56/README_pejacevic.txt 0000664 0000000 0000000 00000013445 12155061511 013716 0 ustar Notes about the piuparts installation on pejacevic.debian.org and it's slave
============================================================================
This document describes the setup for http://piuparts.debian.org - it's used
for reference for the Debian System Administrators (DSA) as well as a guide
for other setting up a similar system, with the piuparts source code
installed from git. For regular installations we recommend to use the
piuparts-master and piuparts-slaves packages as described in
/usr/share/doc/piuparts-master/README_server.txt
== Installation
piuparts.debian.org is a setup running on two systems: pejacevic.debian.org,
running the piuparts-master instance and an apache webserver to display the
results and piu-slave-bm-a.debian.org, running four piuparts-slave nodes.
=== piuparts installation from source
* basically, apt-get build-dep piuparts - in reality both systems get their
package configuration from git.debian.org/git/mirror/debian.org.git
* pejacevic runs a webserver as well (see below for apache configuration)
* Copy 'http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git;hb=master;a=blob_plain;f=update-piuparts-master-setup'
and 'http://anonscm.debian.org/gitweb/?p=piuparts/piuparts.git;hb=master;a=blob_plain;f=update-piuparts-slave-setup'
to the hosts which should be master and slave. (It's possible and has been
done for a long time to run them on the same host.(
Run the scripts as the piupartsm and piupartss users and clone that git
repositry into '/srv/piuparts.debian.org/src' in the first place. Then
checkout the bikeshed branch.
* See below for further user setup instructions.
* Provide '/srv/piuparts.debian.org' - on the slave ideally with a 'tmp'
directory which is on tmpfs.
* `sudo ln -s /srv/piuparts.debian.org/etc/piuparts /etc/piuparts`
=== User setup
On pejacevic the piuparts-master user piupartsm needs to be installed, on
piu-slave-bm-a a piupartss user is needed for the slave.
Both are members of the group piuparts and '/srv/piuparts.debian.org' needs to
be chmod 2775 and chown piuparts(sm):piuparts.
==== '~/bashrc' for piupartsm and piupartss
Do this for the piupartsm user on pejacevic and piupartss on the slave:
----
piupartsm@pejacevic$ cat >> ~/.bashrc <<-EOF
# added manually for piuparts
umask 0002
export PATH="~/bin:\$PATH"
EOF
----
==== set up ssh pubkey authentification
Then create an SSH keypair for piupartss and put it into
'/etc/ssh/userkeys/piupartsm' on pejacevic, so the piupartss user can login
with ssh and run only piuparts-master. Restrict it like this:
----
$ cat /etc/ssh/userkeys/piupartsm
command="/srv/piuparts.debian.org/share/piuparts/piuparts-master",from="2001:41c8:1000:21::21:7,5.153.231.7",no-port-forwarding,no-X11-forwarding,no-agent-forwarding ssh-rsa ...
----
=== Setup sudo
This is actually done by DSA:
==== '/etc/sudoers' for pejacevic
----
#piuparts admins
%piuparts ALL=(piupartsm) ALL
----
==== '/etc/sudoers' for piu-slave-bm-a
----
# The piuparts slave needs to handle chroots.
piupartss ALL = NOPASSWD: ALL
#piuparts admins
%piuparts ALL=(piupartss) ALL
----
=== Apache configuration
Any other webserver will do but apache is used on pejacevic (and maintained by DSA):
----
ServerName piuparts.debian.org
ServerAdmin debian-admin@debian.org
ErrorLog /var/log/apache2/piuparts.debian.org-error.log
CustomLog /var/log/apache2/piuparts.debian.org-access.log combined
DocumentRoot /srv/piuparts.debian.org/htdocs
AddType text/plain log
HostnameLookups Off
UseCanonicalName Off
ServerSignature On
UserDir disabled
# vim:set syn=apache:
----
== Running piuparts
=== Updating the piuparts installation
Updating the master, pejacevic.debian.org:
----
holger@pejacevic$ sudo su - piupartsm update-piuparts-master-setup bikeshed origin
----
Updating the slave, piu-slave-bm-a.debian.org:
----
holger@piu-slave-bm-a$ sudo su - piupartss update-piuparts-slave-setup bikeshed origin
----
=== Running piuparts
==== Starting and stopping the slaves
Run the following script under *your* user account to start four instances of
piuparts-slave on pejacevic, piuparts-master will be started automatically by
the slaves.
----
holger@pejacevic:~$ sudo -u piupartss -i slave_run
----
There are several cronjobs installed via '~piupartsm/crontab' and
'~piupartss/crontab') which monitor the slaves and the hosts it's running on.
It's possible to kill a slave any time by pressing Ctrl-C.
Pressing Ctrl-C once will wait for the current test to finish,
pressing twice will abort the currently running test (which will be redone).
Clean termination may take some time and can be aborted by a third Ctrl-C,
but that may leave temporary directories and processes around.
See the 'piuparts_slave_run (8)' manpage for more information on 'slave_run'.
==== Joining an existing slave session
Run the following script under *your* user account:
----
holger@pejacevic:~$ sudo -u piupartss -i slave_join
----
See the 'piuparts_slave_join (8)' manpage for more information on 'slave_join'.
=== Filing bugs
Use the following usertags:
----
User: debian-qa@lists.debian.org
Usertags: piuparts piuparts.d.o
----
=== Generating reports for the website
'piuparts-report' is run daily at midnight and at noon from
'~piupartsm/crontab' on pejacevic.
=== Cronjobs to aid problem spotting
Some cronjobs to aid problem spotting reside in '~piupartsm/bin/' and are run
daily by '~piupartsm/crontab'.
- 'detect_network_issues' should detect failed piuparts runs due to network
issues on the host.
- 'detect_stale_mounts' should detect stale mountpoints (usually of /proc)
from failed piuparts runs.
More checks should be added as we become aware of them.
== Authors
Last updated: May 2013
Holger Levsen
// vim: set filetype=asciidoc:
piuparts-0.56/TODO 0000664 0000000 0000000 00000015504 12253565512 010706 0 ustar Things to do for piuparts
=========================
Please also check the BTS, especially the bugs with a severity higher than
wishlist!
for 0.5x:
- Create links in htdocs from $codename to stable/testing/unstable used
for all the sections. (Closes: #696093)
Create simple master cronjob to this daily.
- use /sys/fs/selinux from wheezy onwards - see #682068 for more info
- make piuparts-master + piuparts-slave packages work with documented steps:
- do not enable sudoers.d/piuparts-slave in p-s.deb, neither the cronjobs.
- this is manual work one has done once after installation. we only
document whats needs to be done how.
- the shipped crontabs are not suitable from cron.d/ due to missing user
column -> provide instructions how to install them as piuparts(s|m) user
from /u/s/doc/p-(s|m)/examples
- verify that all scripts in $user/bin/ also work from master-slave packages
eventually enable some conjobs in the packages
- slave.postinst should setup '~piupartsm/.ssh/authorized_keys' using triggers
dpkg-trigger piuparts-master-please-install-the-slave-key
+ the current method only works if -slave is configured after -master
- maybe do this manually again as well?
- Documentation related:
- examples are duplicated in piuparts.1.txt and README.txt - only keep one copy.
- README_server has some duplication information on configuration as well.
- more stats and graphs:
- packages processed per day and section
- master writes submissions.txt per section since 0.45
- # of open bugs with tag piuparts
- generate http://piuparts.debian.org/stable/states.png + testing.png from
existing data
- generate simple diagrams: number of source + binary package in all single distros:
squeeze, wheezy, jessie, sid.
- move counts.txt from htdocs to master.
- add a sample config with all possible keys set to some useful value
(like /usr/share/doc/apt/examples/configure-index.gz)
- piuparts.conf.pejacevic: use mirror via nfs (faster)
- look for a solution to use the global debian mirror for debian-backports,
too, to avoid hardcoding a specific mirror in distros.conf
- install from git/Makefile: remove the need for /etc/piuparts
- maybe compress all logfiles
- link wiki:piuparts/UseCases||UsageTips in README_1st (after we agreed on a name)
for 0.6x:
- enable unit tests again.
- if it weren't for 'slave-bin/slave_cleanup', the slave would only need
rights to run "sudo piuparts" but nothing else. If we can clean this up,
the sudoers.d should recommend sudo (lsof|kill|umount) for admins.
- if there were real schroot support, piuparts could be used without sudo.
(#708663)
- generate piuparts.1.txt automatically from piuparts.py - see this blog post
for a nice howto:
http://andialbrecht.wordpress.com/2009/03/17/creating-a-man-page-with-distutils-and-optparse/
- though this seems pretty complicated... maybe rather grep for
parser.add_option and help= in piuparts.py ?!
- requires merging all the additional infomation in piuparts.1.txt into piuarts.py
- parsing piuparts --help output may be easier than parsing piuparts.py
- rework known_problems:
- split detect_well_known_errors
- parsing the logfiles should stay there
- generating the html should be integrated into piuparts-report
- use a number prefix for sorting
- add title information
- piuparts-report: "discover" the available known_problems, dont hardcode the
list
- drop _issue/_error duplication, have flags inside to indicate thether to
generate _issues.tpl (pass/) and/or _error.tpl (fail/ bugged/ affected/)
- rework known problems to a python-friendlier format
- accept a PIUPARTS_CONF environment variable everywhere to point to a different
piuparts.conf
- write reportbug-like wrapper for mass bug filing (start simple, make it more
sophisticated later).
- rewrite piuparts-analyze to run over all sections and cache BTS responses
- "decorate" (strike-through) bug links generated by piuparts-analyze to
indicate resolved state (take package version into account!)
- verify that find_default_debian_mirrors does something sane
- find_default_debian_mirrors: if parts[2] contains a / (think stable/updates
for security.d.o), you can't ignore this, it will break later...
- with distros.conf this may no longer be needed
- check whether find_default_debian_mirrors produces something useful if
sources.list does not exist (and sources.list.d/*.list is there instead)
- maybe parse 'apt-cache policy' to get the mirror list instead
- make it possible to call aptitude (or similar) instead of apt-get and allow to
override the commandline arguments.
- the templates used by update-reports.py and detect_well_known_errors should
be taken from /etc/piuparts/templates/ and not be included in the python source
- check the logfiles (especially pass/) for
- "Exception in thread"
- java stacktraces
- "Can't locate .* in @INC"
for 0.7x and later:
- mounting /proc and perhaps others (usbfs, sysfs, /dev/pts, etc.) in the chroot
might be a good idea because some packages might need this.
- report:
- write stats about the reasons for failures, as its done with shell scripts
now (piuparts-analyze.py is an existing "fragment".)
- RSS feeds of logs
- do more fancy R graphs, eg. also per state
- link (and target) to piuparts.d.o configuration is static to pejacevic. should
refer to the actual hosts configuration if running somewhere else
- a redirect of http://piuparts.d.o/foo to http://p.d.o/source/f/foo.html would
be nice
- really support multiple architectures:
- piuparts-report should have a list of available arch and list packages only
available on untested archs in a new state
"(depends-)not-available-on-tested-archs"
- master should (per default) only schedule packages which are not available
on the master arch to slaves of different archs ->
"schedule-evenly-to-slaves = no"
- piuparts-master: keep track of to whom a reservation was given
- piuparts can't currently test upgrades of required packages. (Because they
cannot be removed, it assumes these are untestable, which is only true for
removal tests...
- all distupgrade tests implicitly tests these upgrades, although not
individually per package
- not sure if it's a sensible thing to to, but provide a way to turn off
debugging output for piuparts.py - see
http://docs.python.org/library/logging.html
- commandline-switches for all programms
- move shell cronjobs functionality into master, slave & report
1.0 should really have automated testing of piuparts itself...
-----------------------------------------------------------------
- create archive of broken packages to provide test cases for piuparts testing.
- create emacspeak-broken-dpkg-preconfigure package for broken repo. (then later
put more broken packages in there and use that for testing piuparts)
piuparts-0.56/piuparts-master-backend.py 0000664 0000000 0000000 00000030527 12253565512 015317 0 ustar #!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright 2005 Lars Wirzenius (liw@iki.fi)
# Copyright © 2011-2013 Andreas Beckmann (anbe@debian.org)
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
# Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
"""Distributed piuparts processing, master program
Lars Wirzenius
"""
import sys
import logging
import ConfigParser
import os
import fcntl
import time
import random
import piupartslib
from piupartslib.packagesdb import LogfileExists
from piupartslib.conf import MissingSection
CONFIG_FILE = "/etc/piuparts/piuparts.conf"
DISTRO_CONFIG_FILE = "/etc/piuparts/distros.conf"
log_handler = None
def setup_logging(log_level, log_file_name):
logger = logging.getLogger()
global log_handler;
logger.removeHandler(log_handler)
if log_file_name:
log_handler = logging.FileHandler(log_file_name)
else:
log_handler = logging.StreamHandler(sys.stderr)
logger.addHandler(log_handler)
logger.setLevel(log_level)
class Config(piupartslib.conf.Config):
def __init__(self, section="master", defaults_section=None):
piupartslib.conf.Config.__init__(self, section,
{
"log-file": None,
"master-directory": ".",
"proxy": None,
"mirror": None,
"distro": None,
"area": None,
"arch": None,
"upgrade-test-distros": None,
"depends-sections": None,
},
defaults_section=defaults_section)
class CommandSyntaxError(Exception):
def __init__(self, msg):
self.args = msg,
class ProtocolError(Exception):
def __init__(self):
self.args = "EOF, missing space in long part, or other protocol error",
class Protocol:
def __init__(self, input, output):
self._input = input
self._output = output
def _readline(self):
line = self._input.readline()
logging.debug(">> " + line.rstrip())
return line
def _writeline(self, line):
logging.debug("<< " + line)
self._output.write(line + "\n")
self._output.flush()
def _short_response(self, *words):
self._writeline(" ".join(words))
def _read_long_part(self):
lines = []
while True:
line = self._input.readline()
if not line:
raise ProtocolError()
if line == ".\n":
break
if line[0] != " ":
raise ProtocolError()
lines.append(line[1:])
return "".join(lines)
class Master(Protocol):
def __init__(self, input, output):
Protocol.__init__(self, input, output)
self._commands = {
"section": self._switch_section,
"recycle": self._recycle,
"idle": self._idle,
"status": self._status,
"reserve": self._reserve,
"unreserve": self._unreserve,
"pass": self._pass,
"fail": self._fail,
"untestable": self._untestable,
}
self._section = None
self._lock = None
self._writeline("hello")
def _init_section(self, section):
if self._lock:
self._lock.close()
# clear all settings from a previous section and set defaults
self._section = None
self._lock = None
self._recycle_mode = False
self._idle_mode = None
self._idle_stamp = os.path.join(section, "idle.stamp")
self._package_databases = None
self._binary_db = None
config = Config(section=section, defaults_section="global")
try:
config.read(CONFIG_FILE)
except MissingSection:
return False
if not os.path.exists(section):
os.makedirs(section)
self._lock = open(os.path.join(section, "master.lock"), "we")
try:
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
return False
self._section = section
logging.debug("switching logfile")
logfile = config["log-file"] or os.path.join(section, "master.log")
setup_logging(logging.DEBUG, logfile)
logging.debug("connected")
# start with a dummy _binary_db (without Packages file), sufficient
# for submitting finished logs
self._binary_db = piupartslib.packagesdb.PackagesDB(prefix=section)
return True
def _init_db(self):
if self._package_databases is not None:
return
self._package_databases = {}
self._load_package_database(self._section)
self._binary_db = self._package_databases[self._section]
def _load_package_database(self, section):
if section in self._package_databases:
return
config = Config(section=section, defaults_section="global")
config.read(CONFIG_FILE)
distro_config = piupartslib.conf.DistroConfig(DISTRO_CONFIG_FILE, config["mirror"])
db = piupartslib.packagesdb.PackagesDB(prefix=section)
if self._recycle_mode and self._section == section:
db.enable_recycling()
self._package_databases[section] = db
if config["depends-sections"]:
deps = config["depends-sections"].split()
for dep in deps:
self._load_package_database(dep)
db.set_dependency_databases([self._package_databases[dep] for dep in deps])
db.load_packages_urls(
distro_config.get_packages_urls(
config.get_distro(),
config.get_area(),
config.get_arch()))
if config.get_distro() != config.get_final_distro():
# take version numbers (or None) from final distro
db2 = piupartslib.packagesdb.PackagesFile()
db2.load_packages_urls(
distro_config.get_packages_urls(
config.get_final_distro(),
config.get_area(),
config.get_arch()))
for package in db.get_all_packages():
if package["Package"] in db2:
package["Version"] = db2[package["Package"]]["Version"]
else:
package["Version"] = "None"
def _clear_idle(self):
if not self._idle_mode is False:
self._idle_mode = False
if os.path.exists(self._idle_stamp):
os.unlink(self._idle_stamp)
def _set_idle(self):
if not self._idle_mode is True:
self._idle_mode = True
open(self._idle_stamp, "w").close()
os.utime(self._idle_stamp, (-1, self._binary_db._stamp))
def _get_idle_status(self):
""" Returns number of seconds a cached idle status is still valid, or 0 if not known to be idle. """
if not os.path.exists(self._idle_stamp):
return 0
stamp_mtime = os.path.getmtime(self._idle_stamp)
ttl = stamp_mtime + 3600 - time.time()
if ttl <= 0:
return 0 # stamp expired
if stamp_mtime < self._binary_db.get_mtime():
return 0 # stamp outdated
return ttl + random.randrange(120)
def do_transaction(self):
line = self._readline()
if line:
parts = line.split()
if len(parts) > 0:
command = parts[0]
args = parts[1:]
if self._section is None and command != "section":
raise CommandSyntaxError("Expected 'section' command, got %s" % command)
if command in self._commands:
self._commands[command](command, args)
return True
else:
raise CommandSyntaxError("Unknown command %s" % command)
return False
def _check_args(self, count, command, args):
if len(args) != count:
raise CommandSyntaxError("Need exactly %d args: %s %s" %
(count, command, " ".join(args)))
def dump_pkgs(self):
for st in self._binary_db.get_states():
for name in self._binary_db.get_pkg_names_in_state(st):
logging.debug("%s : %s\n" % (st, name))
def _switch_section(self, command, args):
self._check_args(1, command, args)
if self._init_section(args[0]):
self._short_response("ok")
elif self._lock is None:
# unknown section
self._short_response("error")
else:
self._short_response("busy")
def _recycle(self, command, args):
self._check_args(0, command, args)
if self._binary_db.enable_recycling():
self._idle_stamp = os.path.join(self._section, "recycle.stamp")
self._recycle_mode = True
self._short_response("ok")
else:
self._short_response("error")
def _idle(self, command, args):
self._check_args(0, command, args)
self._short_response("ok", "%d" % self._get_idle_status())
def _status(self, command, args):
self._check_args(0, command, args)
self._init_db()
stats = ""
if self._binary_db._recycle_mode:
stats += "(recycle) "
total = 0
for state in self._binary_db.get_states():
count = len(self._binary_db.get_pkg_names_in_state(state))
total += count
stats += "%s=%d " % (state, count)
stats += "total=%d" % total
self._short_response("ok", stats)
def _reserve(self, command, args):
self._check_args(0, command, args)
self._init_db()
package = self._binary_db.reserve_package()
if package is None:
self._set_idle()
self._short_response("error")
else:
self._clear_idle()
self._short_response("ok",
package["Package"],
package["Version"])
def _unreserve(self, command, args):
self._check_args(2, command, args)
self._binary_db.unreserve_package(args[0], args[1])
self._short_response("ok")
def _pass(self, command, args):
self._check_args(2, command, args)
log = self._read_long_part()
try:
self._binary_db.pass_package(args[0], args[1], log)
except LogfileExists:
logging.info("Ignoring duplicate submission: %s %s %s"
% ("pass", args[0], args[1]))
self._short_response("ok")
def _fail(self, command, args):
self._check_args(2, command, args)
log = self._read_long_part()
try:
self._binary_db.fail_package(args[0], args[1], log)
except LogfileExists:
logging.info("Ignoring duplicate submission: %s %s %s"
% ("fail", args[0], args[1]))
self._short_response("ok")
def _untestable(self, command, args):
self._check_args(2, command, args)
log = self._read_long_part()
try:
self._binary_db.make_package_untestable(args[0], args[1], log)
except LogfileExists:
logging.info("Ignoring duplicate submission: %s %s %s"
% ("untestable", args[0], args[1]))
self._short_response("ok")
def main():
setup_logging(logging.INFO, None)
global_config = Config(section="global")
global_config.read(CONFIG_FILE)
if global_config["proxy"]:
os.environ["http_proxy"] = global_config["proxy"]
master_directory = global_config["master-directory"]
if not os.path.exists(master_directory):
os.makedirs(master_directory)
os.chdir(master_directory)
m = Master(sys.stdin, sys.stdout)
while m.do_transaction():
pass
logging.debug("disconnected")
if __name__ == "__main__":
main()
# vi:set et ts=4 sw=4 :
piuparts-0.56/conf/ 0000775 0000000 0000000 00000000000 12253565512 011136 5 ustar piuparts-0.56/conf/piuparts.conf.sample 0000664 0000000 0000000 00000002057 12144653075 015141 0 ustar #
# This is the configuration file for piuparts running in master-slave mode.
#
# You MUST make sure that master-host, master-user, master-directory, and
# mirror are set correctly.
#
[global]
sections = sid
mirror = http://cdn.debian.net/debian
master-host = localhost
master-user = piupartsm
piuparts-command = sudo piuparts --scriptsdir /etc/piuparts/scripts
master-directory = /var/lib/piuparts/master
slave-directory = /var/lib/piuparts/slave
basetgz-directory = /var/cache/piuparts/basetgz
output-directory = /var/lib/piuparts/htdocs
tmpdir = /var/cache/piuparts/tmp
doc-root = /piuparts/
idle-sleep = 300
max-tgz-age = 604800
max-reserved = 50
expire-old-days = 120
reschedule-old-days = 90
reschedule-old-count = 150
expire-fail-days = 45
reschedule-fail-days = 30
reschedule-fail-count = 25
[sid]
precedence = 1
description = "Debian sid / main"
piuparts-flags = --no-symlinks
distro = sid
# area = main
# arch = amd64
upgrade-test-distros =
debug = no
## another example:
## [s-p-u-i386]
## distro = stable-proposed-updates
## # area = main
## arch = i386
piuparts-0.56/conf/piuparts.apache 0000664 0000000 0000000 00000000260 12144650665 014151 0 ustar Alias /piuparts /var/lib/piuparts/htdocs
Order allow,deny
Allow from all
Options indexes
DefaultType text/plain
piuparts-0.56/conf/crontab-slave.in 0000664 0000000 0000000 00000000774 12144650665 014241 0 ustar # m h dom mon dow (0|7=sun,1=mon) command
#
# start slave_run automatically after reboots
#
@reboot @sharedir@/piuparts/slave/slave_cleanup ; @sharedir@/piuparts/slave/slave_run
#
# monitor slave session every hour
#
16 * * * * @sharedir@/piuparts/slave/detect_slave_problems
#
# monitor for problems every six hours
#
23 */6 * * * @sharedir@/piuparts/slave/detect_leftover_processes
#
# monitor for cruft in the temporary directory once a day
#
00 0 * * * @sharedir@/piuparts/slave/detect_tmp_cruft
piuparts-0.56/conf/piuparts.sudoers 0000664 0000000 0000000 00000000247 12227526574 014424 0 ustar # The piuparts slave needs to handle chroots.
piupartss ALL = NOPASSWD: ALL
#piuparts admins
%piuparts ALL=(piupartss) ALL
%piuparts ALL=(piupartsm) ALL
piuparts-0.56/conf/crontab-master.in 0000664 0000000 0000000 00000001762 12155061511 014404 0 ustar # m h dom mon dow (0|7=sun,1=mon) command
#
# cleanup $HTDOCS/daily.lock
#
@reboot @sharedir@/piuparts/master/master_cleanup
#
# generate reports twice a day
# (dinstall runs 1|7|13|19:52, so this is long after mirror pushes...)
#
0 0,12 * * * @sharedir@/piuparts/master/generate_daily_report
#
# reschedule old logs twice a day
#
00 3,15 * * * @sharedir@/piuparts/master/reschedule_oldest_logs
#
# monitor for problems once a day
# - these may result in packages being retested
# - if that's not the case, run them from within generate_daily_report
#
30 3-21/6 * * * @sharedir@/piuparts/master/detect_network_issues
45 3-21/6 * * * @sharedir@/piuparts/master/detect_piuparts_issues
0 22 * * * @sharedir@/piuparts/master/detect_archive_issues
30 22 * * * @sharedir@/piuparts/master/report_untestable_packages
0 23 * * * @sharedir@/piuparts/master/report_stale_reserved_packages
#
# misc
#
0 2 * * * @sharedir@/piuparts/master/prepare_backup
55 23 * * * @sharedir@/piuparts/master/gather_bts_stats
piuparts-0.56/conf/distros.conf 0000664 0000000 0000000 00000004765 12253565512 013510 0 ustar #
# Omitted fields will be set to defaults and completely "missing"
# [distribution] entries will automatically be generated as follows:
#
# []
# uri =
# distribution =
# components =
# depends =
# candidates =
# target-release =
#
# These are the standard fields for sources.list entries:
# uri, distribution, components
#
# A non-empty "target-release" will add a -t argument to apt-get:
# apt-get -t ...
#
# The "depends" entry can be used to refer to the "parent distribution"
# (e.g. stable) of a "partial distribution" (e.g. stable-backports).
# These are resolved recursively and will be made available in the
# sources.list file.
#
# The "candidates" entry can be used to build a (virtually) merged
# Packages file from one or more partial distribution (and maybe a full
# parent distribution). This is used for selecting (package,version)
# tuples to be tested. No recursive lookup.
[etch]
uri = http://archive.debian.org/debian
[lenny]
uri = http://archive.debian.org/debian
[squeeze/updates]
uri = http://security.debian.org
depends = squeeze
[squeeze-updates]
depends = squeeze
[squeeze-proposed-updates]
depends = squeeze
[squeeze-proposed]
uri = None
depends = squeeze squeeze/updates squeeze-updates squeeze-proposed-updates
candidates = squeeze squeeze/updates squeeze-proposed-updates
[squeeze-backports]
uri = http://ftp.de.debian.org/debian-backports
depends = squeeze squeeze/updates
target-release = squeeze-backports
[squeeze-backports-sloppy]
uri = http://ftp.de.debian.org/debian-backports
depends = squeeze squeeze-backports
target-release = squeeze-backports-sloppy
[wheezy/updates]
uri = http://security.debian.org
depends = wheezy
[wheezy-updates]
depends = wheezy
[wheezy-proposed-updates]
depends = wheezy
[wheezy-proposed]
uri = None
depends = wheezy wheezy/updates wheezy-updates wheezy-proposed-updates
candidates = wheezy wheezy/updates wheezy-proposed-updates
[wheezy-backports]
depends = wheezy wheezy/updates
target-release = wheezy-backports
[jessie-proposed-updates]
depends = jessie
# alias
[testing]
distribution = jessie
[experimental]
depends = sid
target-release = experimental
# It's also possible to have "virtual" entries by setting uri to
# the string "None". This allows e.g. to combine several partial
# distributions. Such virtual distros can be used for standalone
# piuparts runs, but not in master-slave setup.
[squeeze-current]
uri = None
depends = squeeze/updates squeeze-updates
piuparts-0.56/CONTRIBUTING 0000664 0000000 0000000 00000001556 12223270751 012045 0 ustar Contributing to this project
----------------------------
It's helpful to track fixes or new features via wishlist bugs against the
'piuparts' package, eg with the 'reportbug' tool ('devscripts' package).
The BTS will ensure the developers' mailing list
piuparts-devel@lists.alioth.debian.org
is notified.
Patches can be submitted by mail (git format-patch, see below)
or as requests to pull from a publicly-visible git repository.
In either case, please make a topic branch based on the 'develop' branch.
You can send patches or requests to the development list,
or to the tracking bug: @bugs.debian.org.
One possible workflow:
git clone git://anonscm.debian.org/piuparts/piuparts.git
git checkout origin/develop -b
git commit -a
git format-patch -M origin/develop
reportbug piuparts
piuparts-0.56/ChangeLog 0000664 0000000 0000000 00000025520 12144650665 011772 0 ustar 2009-12-18 Holger Levsen
* This changelog is obsolete since several years. See debian/changelog
instead.
2005-11-12 Lars Wirzenius
* Further detailed changes to be seen in the bzr commit message
log, and summarized in the NEWS file for each release.
2005-10-17 Lars Wirzenius
* Version 0.12.
* /var/log/faillog ignored by default.
* Documented that upgrade testing to experimental doesn't work.
2005-10-14 Lars Wirzenius
* Version 0.11.
* Summary of changes:
- Checks for missing package names on command line immediately,
not after creating chroot.
- apt-get in the chroot gets configured to allow unauthenticated
repositories.
- Tweaks to list of files ignored by default.
2005-09-15 Lars Wirzenius
* Version 0.10.
2005-09-15 Lars Wirzenius
* piuparts.py: When creating policy-rc.d, make it executable.
* piuparts.docbook: Added missing "if". Thanks, Jonas Meurer.
* run-piuparts.py: Wrote.
2005-09-09 Lars Wirzenius
* Version 0.9.
2005-09-08 Lars Wirzenius
* piuparts.py: Added --resolv-deps to debootstrap call, so that it
works with testing/sid.
2005-09-05 Lars Wirzenius
* piuparts.py, piuparts.docbook: Added -k (--keep-tmpdir) option.
* piuparts.py: Added some more default ignores.
* piuparts.py: Remember which packages own which files before
removing packages.
2005-08-31 Lars Wirzenius
* piuparts.py: Made panic() work. Added feature to report which
packages own the new/modifed/removed files. Requested in Debian
bug 324248 by Anibal Monsalve Salazar.
2005-08-30 Lars Wirzenius
* piuparts.py: Use Python 2.3's logging module instead of NIH
code. Also, log everything identically to stdout and log file,
since the old behavior was confusing to many people, as observed
on IRC and at Debconf5. This also obsoletes -v (--verbose).
* piuparts.docbook: Document that -v (--verbose) is obsolete.
2005-08-30 Lars Wirzenius
* piuparts.docbook: It's , not , and you need to
pay attention to warning messages, you imbecile.
2005-08-29 Lars Wirzenius
* Version 0.8.
2005-08-29 Lars Wirzenius
* piuparts.py: Accept matches for any part of a filename, instead
of requiring that they match everything. This is less surprising
to users and they can still use ^ and $ to anchor a match to the
ends, if they need it.
* Makefile: Ignore the fdmount temp files that cause upgrades to
etch/sid fail at the moments, and re-enable upgrade tests.
* piuparts.py, piuparts.docbook: Added option -t and changed the
default location for temporary files and directorys (including
the chroot) to be $TMPDIR (or /tmp if not set).
2005-08-25 Lars Wirzenius
* Makefile: Upgrade testing from sarge to etch fails for me, because
of something to do with fdutils, so I have disabled it for now.
* piuparts.py: Only use the first mirror found in sources.list,
which is anyway what the manual page claims we are doing. This
should reduce the problems people have been having with piuparts
creating broken source.lists in the chroot when people have more
than a canonical Debian repository listed.
* piuparts.py: Remove temporary files upon error exit.
* piuparts.docbook: Use code names instead of
stable/testing/unstable in the example.
2005-07-15 Lars Wirzenius
* Version 0.7.
2005-07-15 Lars Wirzenius
* Makefile: Moved binary to sbin since only root can usefully
run it.
2005-07-13 Lars Wirzenius
* piuparts.py: Create a /usr/sbin/policy-rc.d inside the chroot
when it is created. Suggested by Frank Lichtenheld.
* piuparts.py: Fixed a message from "FAIL:" to "PASS:". Oops.
* Makefile: Commented out an explicit mirror definition so it
should now use sources.list defaults.
2005-07-10 Lars Wirzenius
* piuparts.py: Suggested by Frank Lichtenheld: don't run dpkg
and do other work if there are no files (cleans up log file,
saves a couple of seconds).
* piuparts.py, piuparts.docbook: Implemented and documented a
limit on the size of outputs of command piuparts runs.
2005-07-08 Lars Wirzenius
* piuparts.py: When a command has been run and it failed, show
the output it gave. Removed extra dashes from argument to
getopt. Changed references to "dit" (the old name for the
program) to "piuparts". Changed some log file lines a bit easier
to grep for, and made the start of a log easier to see. Bugs
reported and wishes expressed by Frank Lichtenheld.
2005-07-08 Lars Wirzenius
* piuparts.docbook: Added notes that piuparts needs to be run as
root and that options must come before other command line
arguments.
2005-07-05 Lars Wirzenius
* Version 0.6.
2005-07-05 Lars Wirzenius
* piuparts.py: Bugfix: when removing packages from the chroot,
don't remove packages being depended on before the packages doing
the depending. That won't work. Any fool would know it, except me.
* piuparts.py: Implemented saving of meta data for the upgrade
testing between Debian releases.
2005-07-04 Lars Wirzenius
* piuparts.py: Made it possible to specify components to use for
a mirror.
* piuparts.docbook: Documented the user visible changes.
2005-06-27 Lars Wirzenius
* piuparts.py: Remove and purge dependencies before the packages
that were installed, so that problems with postrm scripts using
non-essential packages upon purge become evident.
2005-06-27 Lars Wirzenius
* piuparts.py, piuparts.docbook: Added -n / --no-ignores option.
2005-06-27 Lars Wirzenius
* piuparts.py: Made mirrors mentioned in /etc/apt/sources.list be
the default mirrors. Also, --mirror can be used multiple times
now.
* piuparts.docbook: Documented this.
* Makefile: log to tmp3.log instead of twice to tmp2.log.
2005-06-27 Lars Wirzenius
* Makefile: Use -f with gzip so it doesn't complain when installing
new version on top of old.
2005-06-27 Lars Wirzenius
* piuparts.py: Added /etc/ld.so.conf to list of ignored files
and changed exim related ignore patterns (adding exim4, for
example).
* piuparts.py: When purging, use --purge and not --remove to
dpkg.
* piuparts.py: Report version and command line arguments at
startup.
* piuparts.py: When calling install_purge_test, report package
list when command line args are package names, instead of empty
list.
2005-06-23 Lars Wirzenius
* Version 0.5.
2005-06-23 Lars Wirzenius
* piuparts.py: Check symlink targets as well.
2005-06-23 Lars Wirzenius
* piuparts.docbook: Added --ignore-regexp to manual page.
2005-06-23 Lars Wirzenius
* TODO: Added.
* Makefile: Added test case for upgrading between distros.
* piuparts.docbook: Rewrote DESCRIPTION and EXAMPLES sections.
* piuparts.py: Added /var/cache/man/index.db and
/var/log/dpkg.log to ignored files. Added regexps for more
versatile ignoring and -I option.
* piuparts.py: Disabled comparison of mtimes of files for now.
* piuparts.py: File listings are now sorted.
* piuparts.py: Added upgrade tests between Debian releases.
2005-06-20 Lars Wirzenius
* piuparts.py: Refactored things more than a bit in preparation
for new functionality.
2005-06-19 Lars Wirzenius
* Version 0.4.
2005-06-19 Lars Wirzenius
* piuparts.docbook, piuparts.py: Added -p option.
* piuparts.docbook: Updated a description of what the program
does.
2005-06-19 Lars Wirzenius
* piuparts.docbook: Documented upgrade testing.
2005-06-19 Lars Wirzenius
* piuparts.py: Added a simple form of upgrade testing.
2005-06-19 Lars Wirzenius
* piuparts.py: Implemented suggestion from Frank Lichtenheld to
do package purginging in two steps: first remove, then purge.
This helps find postrm scripts that use things they aren't
allowed to use.
* README: Added naming credit to Tollef Fog Heen.
2005-06-18 Lars Wirzenius
* Version 0.3.
2005-06-18 Lars Wirzenius
* piuparts.py: Added /var/cache/debconf and
/var/cache/debconf/passwords.dat to the list of filenames that
are automatically ignored.
* piuparts.py: After unpacking a base tarball, run apt-get update
and clean on it, to be sure everything is up to date.
2005-06-18 Lars Wirzenius
* piuparts.py: Added option -a (--apt) that lets the user install
the packages to be tested via apt-get, instead of by specifying
package files.
* piuparts.docbook: Documented -a (--apt).
2005-06-17 Lars Wirzenius
* piuparts.docbook: Added a note that piuparts is meant for people
making Debian packages before they upload them.
2005-06-17 Lars Wirzenius
* piuparts.py, piuparts.docbook: Added -V (--version) option.
2005-06-17 Lars Wirzenius
* piuparts.py, README: Removed notice about python-apt / apt_pkg
since it wasn't actually used.
* piuparts.py: Changed things so that we us os.walk and os.stat
to find filesystem object meta data, instead of creating a
tarball and then scanning that. This makes things quite a bit
faster: with a pre-generate base tarball (from pbuilder, for
example), a simple installation test can now happen in less than
7 seconds wall clock time.
2005-06-14 Lars Wirzenius
* Version 0.2.
* Second public release. The first one was just a mention on an
IRC channel, so it doesn't really count, and I didn't feel like
doing change log entries before now. This version only does
simple installation and removal testing, but should work at
least for simple cases.
piuparts-0.56/htdocs/ 0000775 0000000 0000000 00000000000 12253565512 011475 5 ustar piuparts-0.56/htdocs/style.css 0000664 0000000 0000000 00000011560 12253565512 013352 0 ustar #main {
border: none;
padding-top: 10px;
padding-bottom: 10px;
padding-left: 10px;
padding-right: 10px;
vertical-align: top;
background-color: #ddd;
}
hr {
width: 100%;
color:#d70751;
background-color:#d70751;
height:2px;
margin: 0px;
padding: 0px;
}
em {
font-weight: bold;
}
p.note {
font-family: sans-serif;
color: #900;
text-align: center;
padding: 5px;
font-size: 11px;
font-weight: normal;
}
div.c1 {text-align: center}
p.text {
font-family: sans-serif;
padding: 5px;
font-weight: normal;
}
body {
padding: 0px;
margin: 0px;
font-family: sans-serif;
font-size: 90%;
color: #000000;
background-color: white;
}
#obeytoyourfriend
{
color: #000;
background-color: white;
border: 0px solid #000;
border-collapse: collapse;
margin-top: 0px;
margin-bottom: 5px;
padding-top: 0px;
padding-left: 15px;
font-family: serif;
font-variant:small-caps;
font-size: 140%;
}
p.validate {
text-align: center;
}
table
{
color: #000;
background-color: #000;
border: 0px solid #000;
border-collapse: separate;
border-spacing: 1px;
}
h1 {
font-size: 140%;
text-align: center;
color: #000;
}
h2 {
font-size: 120%;
text-align: center;
color: #000;
}
h3 {
font-size: 110%;
text-align: center;
color: #000;
}
p {
font-size: 100%;
text-align: justify;
}
tr {
background-color: #FFF;
}
tr.odd {
background-color: #FFFFFF;
}
tr.even {
background-color: #e8e8e8;
}
td.sid {
color: #000;
text-align: left;
}
tr.experimental {
color: #cc0000;
}
tr.unstable {
color: #345677;
}
tr.sid_odd {
color: #000;
}
td.exp {
color: #cc0000;
text-align: left;
}
tr.exp_odd {
color: #900;
}
th {
font-size: 120%;
text-align: center;
font-weight: bold;
background-color: #BDF;
border: 0px solid #000;
padding-top: 10px;
padding-bottom: 10px;
padding-left: 6px;
padding-right: 6px;
}
th.reject {
font-size: 120%;
text-align: center;
font-weight: bold;
background-color: #BDF;
border: 0px solid #000;
padding-top: 10px;
padding-bottom: 10px;
padding-left: 6px;
padding-right: 6px;
}
td {
font-size: 100%;
border: 0px solid #000;
padding: 4px;
padding-left: 6px;
padding-right: 6px;
}
a.needs-bugging {
background-color:#e0c0d0;
}
span.needs-bugging {
background-color:#e0c0d0;
}
#needs-bugging {
background-color:#e0c0d0;
}
a.bugged {
background-color:#c0c0f0;
}
a:link {
color: #0000FF;
text-decoration: none;
}
a:visited {
color: #800080;
text-decoration: none;
}
a:active {
color: #FF0000;
text-decoration: none;
}
a:hover {
color: #0000FF;
text-decoration: underline;
}
#header {
padding: 3px;
}
h1.header {
font-size: 32px;
text-align: left;
margin-bottom: 0px;
margin-top: 0px;
padding-top: 0px;
padding-bottom: 0px;
padding-left: 15px;
}
#footer {
padding: 15px;
font-size: 90%;
text-align: center;
}
/* the following layout is taken from the PTS */
/* Tables */
table.containertable {
clear: both;
background-color: #ddd;
border: none;
}
table.lefttable {
width: 100%;
border-collapse: collapse;
border: 2px solid black;
background-color: white;
color: black;
}
table.righttable {
width: 100%;
border-collapse: collapse;
border: 2px solid black;
background-color: white;
color: black;
}
td.containercell {
background-color: #ddd;
}
td.titlecell {
color: white;
background-color: #d70751;
font-weight: bold;
text-align: center;
padding: 0.2em 0.2em 0.1em 0.2em;
border-top: 3px solid #999;
border-bottom: 1px solid #999;
}
td.alerttitlecell {
color: white;
background-color: #0755d7;
font-weight: bold;
text-align: center;
padding: 0.2em 0.2em 0.1em 0.2em;
border-top: 3px solid #999;
border-bottom: 1px solid #999;
}
td.labelcell {
vertical-align: top;
text-align: left;
padding: 0.2em 0.3em 0.2em 0.3em;
border-bottom: 1px dotted #999;
font-weight: bold;
}
td.alertlabelcell {
color: white;
background-color: #0755d7;
vertical-align: top;
text-align: left;
padding: 0.2em 0.3em 0.2em 0.3em;
border-bottom: 1px dotted #999;
font-weight: bold;
}
td.lightalertlabelcell {
color: white;
background-color: #90c0ff;
vertical-align: top;
text-align: left;
padding: 0.2em 0.3em 0.2em 0.3em;
border-bottom: 1px dotted #999;
font-weight: bold;
}
td.labelcell2 {
padding: 0.2em 0.2em 0.1em 0.2em;
border-top: 1px solid black;
border-right: 1px dotted black;
font-weight: bold;
}
td.contentcell {
text-align: center;
padding: 0.2em 0.3em 0.2em 0.3em;
border-bottom: 1px dotted #999;
}
td.lastcell {
font-size: 80%;
text-align: center;
padding: 0.2em 0.3em 0.2em 0.3em;
border-bottom: 1px dotted #999;
}
td.contentcell2 {
text-align: left;
vertical-align: top;
padding: 0.2em 0.3em 0.2em 0.3em;
border-bottom: 1px dotted #999;
border-right: 1px dotted #999;
}
piuparts-0.56/htdocs/index.tpl 0000664 0000000 0000000 00000064643 12175701021 013330 0 ustar
About piuparts.debian.org
piuparts is a tool for testing that .deb packages can be installed, upgraded, and removed without problems.
piuparts is short for "p ackage i nstallation,
up grading a nd r emoval t esting s uite" and is
a variant of something suggested by Tollef Fog Heen.
It does this by creating a minimal Debian installation in a chroot, and installing,
upgrading, and removing packages in that environment, and comparing the state of the directory tree before and after.
piuparts reports any files that have been added, removed, or modified during this process.
piuparts is meant as a quality assurance tool for people who create .deb packages to test them before they upload them to the Debian package archive.
A quick introduction is available in the piuparts README_1st , and all the options are listed on the piuparts manpage .
To make sure piuparts is run on all packages in Debian, piuparts.debian.org has been set up to run piuparts in master/slave mode . This setup currently consists of two hosts: pejacevic.debian.org and piu-slave-bm-a.debian.org :
pejacevic acts as the piuparts-master, which is responsible for scheduling test jobs to the slaves. The other main task is to generate the reports which are served via http://piuparts.debian.org.
piu-slave-bm-a runs four piuparts-slave instances, which then run piuparts itself.
These hosts run as virtualized hardware on this nice cluster hosted at Bytemark .
To learn more about this setup, follow the "Documentation" links in the navigation menu on the left. Read those READMEs. The piuparts configuration for all the different suite(-combination)s that are currently being tested is also linked there.
Besides all the information provided here (which is sourced in GIT ) there are also three wiki pages on wiki.debian.org currently: the main one which is mostly about
piuparts development , some
frequently asked questions and some
use cases .
Join #debian-qa on irc.debian.org or post on the piuparts development mailinglist if you want to help. The best way to help is to provide patches via GIT pull requests.
These pages are updated twice a day.
News
2013-07-16 To better track bugs in piuparts.debian.org and piuparts itself, a new pseudo-package was created in the BTS: piuparts.debian.org , which will be used for tracking all issues with the piuparts.debian.org service.
2013-06-05 In preparation of the first wheezy point release, another new suite was added: squeeze2wheezy-proposed , testing installation in squeeze, then upgrade to wheezy-proposed-upgrades, ending in purge as usual.
2013-05-30 Another new suite added: wheezy2proposed , testing installation in wheezy, then upgrade to wheezy-proposed-upgrades, ending in purge as usual.
2013-05-29 Another new suite added: squeeze2bpo-sloppy , testing the upgrade from squeeze to squeeze-backports-sloppy, ending in purge as usual.
2013-05-22 The webpages served by http://piuparts.debian.org are updated twice a day now. Further changes which were applied last week: debsums failures have been reenabled, adequate is now run by piuparts (see #703902 ) and two new suites were added: experimental and sid-nodoc , which tests sid without files in /usr/share/doc/<package>.
2013-05-14 Thanks to the new "hardware", piu-slave-bm-a is running four slaves now. Plus, these slaves are also considerably faster than piatti. And there are two new suites being tested: wheezy2jessie and wheezy2bpo2jessie - whoohoo!
2013-05-13 piuparts.debian.org has been moved to a new hardware and hosting location, now running virtualized on this nice cluster at Bytemark. Thanks to the Debian System Administrators for their assistence in setting up the host and maintaining the Debian infrastructure! Also many thanks and kittos to the Department of Computer Science at the University of Helsinki, Finland, for hosting piatti.debian.org since 2006 (at least)! For maintaining this setup we use the *bikeshed* git branch.
2013-03-15 Among many other new features the 0.50 release offers much greater flexibility for configuring and selecting (partial) suites and different mirrors.
Therefore it is possible to test nearly arbitrary upgrade pathes. On piuparts.debian.org this is now used for testing squeeze2bpo2wheezy and sid2experimental . Thanks to Andreas Beckmann for this great new feature!
2013-03-02 While the piuparts.git repo on Alioth will continue to be the main repo, there is also a piuparts clone on github , for those who prefer to send pull requests that way.
2012-06-21 piuparts 0.45 has been released, featuring piuparts-master and piuparts-slave packages to ease installation of such a setup. If you run piuparts in master/slave mode, please let us know.
2012-06-04 Wheezy freeze is approaching and lots of uploads happening. Old piatti hardware has problems keeping up with the pace of uploads, number of packages and distros being tested! :-) Piatti is about six years old...
2012-03-31 Disable lenny2squeeze tests, as lenny has been archived.
2012-03-05 : temporarily disabled this again until we've sorted out problems with it.
2012-02-20 : piuparts-analyse now sends commands the BTS: if a bug has not been explicitly marked fixed in the new version, it can rather very savely be assumed it's still present.
2012-01-30 : Add new suite to be tested, testing2sid, to catch upgrade problems before they reach testing.
2012-01-22 : Since some weeks, piuparts-analyse is captable of moving logfiles from fail to bugged, if there is a bug report usertagged 'piuparts' against that package+version combination. Thus, since today there is a webpage, explaining how to file bugs based on tests run on piuparts.debian.org . So now the question how to help can easily be answered: read that page and start filing bugs!
2012-01-20 : As squeeze2wheezy has been fully tested by today, re-enable rescheduling of old logs for sid, wheezy and squeezewheezy: 200 successful logs older than 90 days are rescheduled each day, plus 25 failed logs older than 30 days.
2011-12-20 : Currently, while the machine is busy testing all of squeeze2wheeezy, all old log rescheduling has been disabled. Normally, these reschedulings happen for sid, wheezy and squeezewheezy: 200 successful logs old than 180 days are rescheduled each day, plus 25 failed logs older than 30 days.
2011-12-10 : Finally, upgrades from squeeze to wheezy are also being tested. Yay!
2011-11-21 : All mails created by the piuparts master-slave setup on piatti.d.o are now sent to the piuparts-reports mailinglist on alioth. Subcribe and learn more about the details of this setup!
2011-10-31 : Re-create base.tgz's every week now, as they will only be replaced if the recreation was successful.
2011-10-23 : piuparts.debian.org is now maintained in git, using the piatti branch.
2011-07-10 : Since today dpkg is run with --force-unsafe-io by for all suites except lenny2squeeze, as dpkg from lenny doesn't support this option.
2011-07-10 : systemd-sysv is the eighth package getting special treatment by piuparts as it needs removal of sysvinit before installation and installation of that package before removal...
2011-04-02 : New daily cronjob to reschedule the oldest 200 logfiles of each sid and wheezy, if they are older then 180 days. IOW: make sure no logfile for sid and wheezy is older than half a year.
2011-02-22 : piatti.debian.org has been upgraded to squeeze.
2011-02-07 : Add wheezy ! Whoohoo! For now, the Wheezy distribution has just been added with the same testing options as Squeeze. In future, squeeze and lenny2squeeze will not be tested anymore, and squeeze2wheezy will also be added...
2011-01-25 : Reschedule 27655 successfully tested packages in Squeeze, since they were tested before the deep freeze. Yesterday all 70 failed and bugged packages were rescheduled too, which surprisingly led to 6 successful tests, followed by a few more dependent packages also being tested.
2011-01-15 : Reschedule 10123 successful and failed logs in lenny2squeeze for re-testing. Those are logs which have been tested before Squeeze was deep frozen or while there was still a bug in piuparts-slave, see last news entry for details.
2011-01-03 : Reschedule 12306 successful and 8 bugged logs in lenny2squeeze for re-testing. Those are logs older than 148 days, which refers to when Squeeze was initially frozen (2010-08-06). Deep freeze was announced on 2010-12-13 and there are 3800 logs older then that too, but for future deletions it's better to use 2010-01-03 (=commit r857), which fixes a bug in piuparts-slave resulting in using the sid packages file for lenny2squeeze tests.
2010-11-28 : debconf-english is the seventh package getting special treatment by piuparts: before removal, debconf-i18n is installed (see #539146 has the details and the news entry for 2010-11-25 lists the other six packages.)
2010-11-26 : Schedule all 159 failed packages in lenny2squeeze for re-testing.
2010-11-25 : Treat six packages specially: sudo (sensibly refuses removal if no root password is set), apt-listbugs (is called by apt and exists if there are RC buggy packages being upgraded), fai-nfsroot, ltsp-client-core (these two packages modify the installed system heavily and thus will only install if conditions are met), file-rc and upstart (these two replace essential packages and therefore apt needs to be told to do this).
2010-11-24 : Disable the logrotate test until #582630 is fixed and reschedule all 51 packages in sid failed due to it.
2010-11-14 : Schedule all 402 failed packages in sid for re-testing.
2010-11-12 : Schedule all 108 failed packages in squeeze for re-testing. (Followup on 2010-09-04.)
2010-11-06 : The lists of known circular depends is now taken from http://debian.semistable.com/debgraph.out.html and maintained separately (and manually) for each tested distribution in piuparts.conf - this is not optimal (which would be piuparts detecting them automatically) but much better than the hardcoded list which we had in the piuparts library since December 2009.
2010-09-04 : Schedule all 27438 passed packages in squeeze for re-testing now that squeeze is frozen.
2009-07-24 : #531349 has been fixed, piuparts results are now displayed in the PTS .
2010-05-18 : From today on, broken logrotate scripts after purge are only reported in sid.
2010-05-16 : Finally enabled testing of sid again. (Actually, sid was enabled on 2010-03-05, but piuparts.d.o was broken until today.)
2010-02-28 : Due to #571925 testing of sid had to be disabled temporarily. On an unrelated note, testing of lenny2squeeze still has some issues atm...
2010-02-25 : Since yesterday, squeeze and lenny2squeeze are being tested with "--warn-on-leftovers-after-purge" making piuparts only warn about leftover files after purge. This has two effects: an decrease in the number of failed logs to process, to better focus on more important problems and second, more packages will be tested, as less packages are (seen as) buggy. Today all failed packages in squeeze and lenny2squeeze have been rescheduled for testing.
2010-02-23 : Since today, piuparts is able to detect broken logrotate scripts after purge, which will need retesting of all successfully tested packages eventually. The failed packages in squeeze also needs retesting, due to split into squeeze and lenny2squeeze last week.
2010-02-16 : The squeeze test has been split into squeeze and lenny2squeeze, where squeeze means package installation in squeeze, removal and purge test, while lenny2squeeze means package installation in lenny, then upgrade to squeeze, then removal and purge test. This allows more issues to be found in squeeze since (potential) brokeness in lenny is not blurring the results in squeeze.
2010-01-05 : Reschedule testing for 319 failed packages in sid and 544 in squeeze, since --warn-on-others is now used.
2009-12-24 : Enable work-in-progress code to enable testing of packages with circular depends. This will allow testing of 5-6000 more packages in sid and squeeze, see #526046 and the 0.39 changelog for details. The list of packages with circular depends is currently hard-coded and will probably become a configuration option but not auto detected. But that's code yet to be written :-)
2009-12-21 : So testing of 13398 in squeeze has taken 12 days, which is no big surprise as the squeeze tests are more complex. Today 499 failed packages from sid and 235 from squeeze have been rescheduled for testing, to catch broken symlinks in those too.
2009-12-12 : After testing 14416 packages in sid in three days, reschedule 15944 packages in squeeze... see previous entry for an explanation why.
2009-12-09 : Reschedule testing for 14287 successfully tested packages in sid, those in squeeze will be rescheduled once all testable package in sid have been tested again. This is because piuparts now creates and maintains chroots securily (using gpg signed Release files for both debootstrap and apt-get) and because it warns if broken symlinks are found in a package.
2009-12-05 : Reschedule testing for ~400 failed packages in sid and ~600 in squeeze, to be followed by a rescheduling of all successful packages. This is because piuparts now warns if broken symlinks are found in a package.
2009-10-08 : Reschedule testing for ~2000 failed packages in sid, which failed because of a problem when minimizing the chroot at the beginning of the piuparts tests. As of today, piuparts running on piuparts.debian.org does not minimize the chroots anymore.
2009-09-18 : Reschedule testing for 17170 (successfully tested) packages in sid, to make sure they still install fine with dependency based booting enabled now in sid. Throwing away 42806 (successful) logfiles from those packages :-)
2009-09-16 : Reschedule testing for 233 failing packages in sid which were affected by #545949 . No packages in squeeze were affected.
2009-06-20 : Failed logs are not grouped into (at the moment) seven types of known errors and one type of issues is detected in successful logs.
2009-06-06 : Reschedule testing for 163 successful and 27 failing packages in sid which were affected by #530501 . Once openssh 1:5.1p1-6 has reached squeeze, this will be done again with 194 packages there.
2009-05-27 : Throw away all failed logs as there was a bug in piuparts leading to use a more uptodate mirror for getting the list of available packages and another for doing the tests. This lead to at least one fixed package which was incorrectly tested as failing, as an old version of the package was tested. To rule out some false positives about 1000 packages will be retested, but on this machine this will only take about a day :-)
2009-05-11 : Filed #528266 and made piuparts ignore files in /tmp after purge. This got rid of 20 failures in sid and 14 in squeeze.
2009-05-06 : Only believe statistics you faked yourself! Up until today piuparts used to include virtual packages (those only exist true the Provides: header) into the calculations of statistics of package states and the total number of packages. Suddenly, sid has 2444 packages less!
2009-05-01 : All packages in squeeze and sid which can be tested have been tested. So it takes about one month to do a full piuparts run against one suite of the archive on this machine, that's almost 1000 packages tested per day.
2009-04-20 : Deleted 86 more failed logfiles (out of 692 failures in total atm) which were due to broken packages, which most likely are temporarily uninstallable issues - a good indicator for this is that all of those failures happened in sid and none in squeeze. For the future there is a cronjob now, to notify the admins daily of such problems. In more distant future those issues should be detected and avoided.
2009-04-18 : Deleted all 14 failed logfiles which complained about /var/games
being present after purge, as this ain't an issue, see #524461 .
2009-04-04 : Deleted all failed logfiles so far for two reasons: until now, only three out of ten failure types where logged with a pattern to search for in the logfiles, now this is done for all ten types of failures. And second, the way of breaking circular dependencies was not bulletproof, thus there were false positives in the failures. Now it should be fine, though maybe this will lead to lots of untestable packages... we'll see.
2009-03-19 : lenny2squeeze is not needed, so all logs for squeeze (as well as lenny2squeeze) were deleted. (As squeeze now includes two kinds of tests: installation and removal in squeeze, and installation in lenny, upgrade to squeeze, removal in squeeze.)
2009-02-28 : Start maintaining piatti.debian.org via the piuparts svn repository on alioth.
2007-02-24 : Holger puts piuparts source in svn .
2006-10-02 : #390754 O: piuparts -- package installation, upgrading and removal testing tool"
2006-09-29 : Lars seeks help maintaining piuparts .
2005-07-05 : #317033 ITP: piuparts -- .deb package installation, upgrading, and removal testing tool
2005-06-19 : Lars writes the first blog post about piuparts (version 0.4).
piuparts-0.56/htdocs/bug_howto.tpl 0000664 0000000 0000000 00000007071 12253565512 014220 0 ustar
How to file bugs based on tests run on piuparts.debian.org
This page shall grow into a well written explaination how to file useful bugs fast. It assumes you are familar with reporting bugs in Debian .
First, of all, read the piuparts logfile and identify why piuparts testing failed.
Then, check the BTS for that package, to see if this issue was already filed as a bug. Often it's also useful to check the source packages bug page.
Then, if there is none, file a bug. If there already is a bug describing the same problem you're seeing in the piuparts logfile, usertag it like this:
User: debian-qa@lists.debian.org
Usertags: piuparts
This will make sure, piuparts.debian.org picks up your bug report (actually, it's piuparts-analyse.py) and marks it as bugged in the database.
Also, if you file a new bug report, don't forget to usertag it! The following is an example bug report. There are many more templates available for you to make use from!
To: submit@bugs.debian.org
Subject: $package: fails to upgrade from 'testing' - trying to overwrite ...
Package: $package
Version: $version
Severity: serious
User: debian-qa@lists.debian.org
Usertags: piuparts
Hi,
during a test with piuparts I noticed your package fails to upgrade from
'testing'. It installed fine in 'testing', then the upgrade to 'sid'
fails because it tries to overwrite other packages files without
declaring a replaces relation.
See policy 7.6 at
http://www.debian.org/doc/debian-policy/ch-relationships.html#s-replaces
From the attached log (scroll to the bottom...):
$useful_except_from_logfile
cheers,
$your_name
attachment: $failed_logfile
Please take care when filing bugs to file meaningful bugs and to not annoy maintainers. Don't nitpick or insist on severities, the important thing is to get the bug fixed, not the right severity. Optionally you can also send copies to the piuparts-devel mailinglist by adding X-debbugs-cc: piuparts-devel@lists.alioth.debian.org pseudo-headers.
Also, you should be aware that what you are doing can probably be seen as mass bug filing (even if you just file a few now, they are part of a series of bugs of one kind) and as such needs to be discussed on debian-devel@lists.d.o first! For many types of bugs this has already been done. This is or should be indicated in the summary web pages as well as the mail templates.
Finally, besides simply usertagging a bug, piuparts-analyse understands more, and you can also do the following:
# assume 987654 is our bug report in buggy-package,
# but the problem only shows up when testing (upgrades of)
# failing-package with piuparts:
bts affects 987654 failing-package
# and if failing-package is from a different source with a different
# version number:
bts found 987654 failing-package/$FAILED_VERSION
piuparts-0.56/htdocs/favicon.ico 0000664 0000000 0000000 00000011146 12144650665 013624 0 ustar F h ( > ( @ ʦ @ ` @ @ @@ @` @ @ @ @ ` ` `@ `` ` ` ` ` @ ` @ ` @ ` @ ` @ @ @ @ @ ` @ @ @ @ @ @ @ @ @ ` @ @ @ @ @@ @@ @@@ @@` @@ @@ @@ @@ @` @` @`@ @`` @` @` @` @` @ @ @@ @` @ @ @ @ @ @ @@ @` @ @ @ @ @ @ @@ @` @ @ @ @ @ @ @@ @` @ @ @ @ @ ` @ ` @ @ @@ @` @ @ @ @ ` ` `@ `` ` ` ` ` @ ` @ ` @ ` @ ` @ ` @ ` @ @ @@ @` @ @ @ @ ` ` `@ `` ` ` ` ` @ ` @ ` @ ` N F FN FF FN FF FF F FF NF FF FFF F WNNFFFF FF NNN FF FF N F F NF W F F F W NF F F F FN F FN FF F FF FF FF FF N FN WFN F FN FFF NFFF FW FFN FF FFN FFF FFF FFFN FFFF FFFF FFFFF NFFFF NFFFFFFFFFFFFFFF FFFFFFFFFFFN NFFNW ??8?{? ( @ ??|??? ( @ ʦ @ ` @ @ @@ @` @ @ @ @ ` ` `@ `` ` ` ` ` @ ` @ ` @ ` @ ` @ @ @ @ @ ` @ @ @ @ @ @ @ @ @ ` @ @ @ @ @@ @@ @@@ @@` @@ @@ @@ @@ @` @` @`@ @`` @` @` @` @` @ @ @@ @` @ @ @ @ @ @ @@ @` @ @ @ @ @ @ @@ @` @ @ @ @ @ @ @@ @` @ @ @ @ @ ` @ ` @ @ @@ @` @ @ @ @ ` ` `@ `` ` ` ` ` @ ` @ ` @ ` @ ` @ ` @ ` @ @ @@ @` @ @ @ @ ` ` `@ `` ` ` ` ` @ ` @ ` @ ` N N N N F VF F NFF F NN F V N V N F F F N N F NF F NFVVNF VFFV ? : + 3J 1X O( <
3'ـ9 5 piuparts-0.56/htdocs/robots.txt 0000664 0000000 0000000 00000000037 12144650665 013551 0 ustar User-agent: *
Disallow: /fail/
piuparts-0.56/htdocs/images/ 0000775 0000000 0000000 00000000000 12144650665 012745 5 ustar piuparts-0.56/htdocs/images/w3c-valid-css.png 0000664 0000000 0000000 00000001362 12144650665 016034 0 ustar PNG
IHDR P pHYs tIME
!ʗ IDATxcLKKcIa̙#ķ,5j
pu [W?y-VMUZ"}Ma _/XЮ`nd/"&;T" s駞g!ޮ=|=+}a[f``Xvlhpl6y/hk200l9|"Z(#Ta|7I`97T!"Ȃ"(ʔ>~wɻOxRE^ZħpP<l5t59XpLI֊{ߙJTȍ4]NJr
JFz|$*+lG)VzE[IW}lU! o~qn'Y%I->x4~g$gKMOUr!,ZPUפ@T+j[U1A@kEwsO?m9|[YV PIK&z*&.Rj T!lHQil)Ir@0ֲPIYWYޒGZo !k IENDB` piuparts-0.56/htdocs/images/valid-html401.png 0000664 0000000 0000000 00000000604 12144650665 015741 0 ustar PNG
IHDR P U sRGB 0PLTE^N;jPPPfffyݮ bKGD H pHYs tIME3f IDAT(cK$`(U 0A(\ynrerd