createrepo-0.10.3/0000755000175000017500000000000012271721201013641 5ustar zpavlaszpavlascreaterepo-0.10.3/modifyrepo.py0000775000175000017500000002370712271721200016405 0ustar zpavlaszpavlas#!/usr/bin/python # This tool is used to manipulate arbitrary metadata in a RPM repository. # Example: # ./modifyrepo.py updateinfo.xml myrepo/repodata # or # ./modifyrepo.py --remove updateinfo.xml myrepo/repodata # or in Python: # >>> from modifyrepo import RepoMetadata # >>> repomd = RepoMetadata('myrepo/repodata') # >>> repomd.add('updateinfo.xml') # or # >>> repomd.remove('updateinfo.xml') # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # (C) Copyright 2006 Red Hat, Inc. # Luke Macken # modified by Seth Vidal 2008 # modified by Daniel Mach 2011 import os import re import sys from createrepo import __version__ from createrepo.utils import checksum_and_rename, compressOpen, MDError from createrepo.utils import _available_compression from yum.misc import checksum, _available_checksums, AutoFileChecksums from yum.repoMDObject import RepoMD, RepoMDError, RepoData from xml.dom import minidom from optparse import OptionParser class RepoMetadata: def __init__(self, repo): """ Parses the repomd.xml file existing in the given repo directory. """ self.repodir = os.path.abspath(repo) self.repomdxml = os.path.join(self.repodir, 'repomd.xml') self.compress_type = _available_compression[-1] # best available if not os.path.exists(self.repomdxml): raise MDError, '%s not found' % self.repomdxml try: self.repoobj = RepoMD(self.repodir) self.repoobj.parse(self.repomdxml) except RepoMDError, e: raise MDError, 'Could not parse %s' % self.repomdxml def _get_mdtype(self, mdname, mdtype=None): """ Get mdtype from existing mdtype or from a mdname. """ if mdtype: return mdtype mdname = os.path.basename(mdname) if re.match(r'[0-9a-f]{32,}-', mdname): mdname = mdname.split('-', 1)[1] return mdname.split('.')[0] def _print_repodata(self, repodata): """ Print repodata details. """ print " type =", repodata.type print " location =", repodata.location[1] print " checksum =", repodata.checksum[1] print " timestamp =", repodata.timestamp print " open-checksum =", repodata.openchecksum[1] print " size =", repodata.size print " open-size =", repodata.opensize def _write_repomd(self): """ Write the updated repomd.xml. """ outmd = file(self.repomdxml, 'w') outmd.write(self.repoobj.dump_xml()) outmd.close() print "Wrote:", self.repomdxml def _remove_repodata_file(self, repodata): """ Remove a file specified in repodata location """ try: fname = os.path.basename(repodata.location[1]) os.remove(os.path.join(self.repodir, fname)) except OSError, ex: if ex.errno != 2: # continue on a missing file raise MDError("could not remove file %s" % repodata.location[1]) def add(self, metadata, mdtype=None): """ Insert arbitrary metadata into this repository. metadata can be either an xml.dom.minidom.Document object, or a filename. """ md = None if not metadata: raise MDError, 'metadata cannot be None' if isinstance(metadata, minidom.Document): md = metadata.toxml() mdname = 'updateinfo.xml' elif isinstance(metadata, str): if os.path.exists(metadata): mdname = os.path.basename(metadata) if mdname.split('.')[-1] in ('gz', 'bz2', 'xz'): mdname = mdname.rsplit('.', 1)[0] oldmd = compressOpen(metadata, mode='rb') else: oldmd = file(metadata, 'r') oldmd = AutoFileChecksums(oldmd, [self.checksum_type]) md = oldmd.read() oldmd.close() else: raise MDError, '%s not found' % metadata else: raise MDError, 'invalid metadata type' ## Compress the metadata and move it into the repodata mdtype = self._get_mdtype(mdname, mdtype) destmd = os.path.join(self.repodir, mdname) if self.compress: destmd += '.' + self.compress_type newmd = compressOpen(destmd, mode='wb', compress_type=self.compress_type) else: newmd = open(destmd, 'wb') newmd.write(md) newmd.close() print "Wrote:", destmd if self.unique_md_filenames: csum, destmd = checksum_and_rename(destmd, self.checksum_type) else: csum = checksum(self.checksum_type, destmd) base_destmd = os.path.basename(destmd) # Remove any stale metadata old_rd = self.repoobj.repoData.pop(mdtype, None) new_rd = RepoData() new_rd.type = mdtype new_rd.location = (None, 'repodata/' + base_destmd) new_rd.checksum = (self.checksum_type, csum) new_rd.size = str(os.stat(destmd).st_size) if self.compress: new_rd.openchecksum = oldmd.checksums.hexdigests().popitem() new_rd.opensize = str(oldmd.checksums.length) new_rd.timestamp = str(int(os.stat(destmd).st_mtime)) self.repoobj.repoData[new_rd.type] = new_rd self._print_repodata(new_rd) self._write_repomd() if old_rd is not None and old_rd.location[1] != new_rd.location[1]: # remove the old file when overwriting metadata # with the same mdtype but different location self._remove_repodata_file(old_rd) def remove(self, metadata, mdtype=None): """ Remove metadata from this repository. """ mdname = metadata mdtype = self._get_mdtype(mdname, mdtype) old_rd = self.repoobj.repoData.pop(mdtype, None) if old_rd is None: print "Metadata not found: %s" % mdtype return self._remove_repodata_file(old_rd) print "Removed:" self._print_repodata(old_rd) self._write_repomd() def main(args): parser = OptionParser(version='modifyrepo version %s' % __version__) # query options parser.add_option("--mdtype", dest='mdtype', help="specific datatype of the metadata, will be derived from the filename if not specified") parser.add_option("--remove", action="store_true", help="remove specified file from repodata") parser.add_option("--compress", action="store_true", default=True, help="compress the new repodata before adding it to the repo (default)") parser.add_option("--no-compress", action="store_false", dest="compress", help="do not compress the new repodata before adding it to the repo") parser.add_option("--compress-type", dest='compress_type', help="compression format to use") parser.add_option("-s", "--checksum", dest='sumtype', help="specify the checksum type to use") parser.add_option("--unique-md-filenames", dest="unique_md_filenames", help="include the file's checksum in the filename, helps with proxies", action="store_true") parser.add_option("--simple-md-filenames", dest="unique_md_filenames", help="do not include the file's checksum in the filename", action="store_false") parser.usage = "modifyrepo [options] [--remove] " (opts, argsleft) = parser.parse_args(args) if len(argsleft) != 2: parser.print_usage() return 0 metadata = argsleft[0] repodir = argsleft[1] try: repomd = RepoMetadata(repodir) except MDError, e: print "Could not access repository: %s" % str(e) return 1 try: # try to extract defaults from primary entry md = repomd.repoobj.getData('primary') sumtype = md.checksum[0] name = os.path.basename(md.location[1]) unique_md_filenames = re.match(r'[0-9a-f]{32,}-', name) != None compress_type = name.rsplit('.', 1)[1] except RepoMDError: sumtype = 'sha256' unique_md_filenames = True compress_type = 'gz' # apply defaults if opts.sumtype is None: opts.sumtype = sumtype if opts.unique_md_filenames is None: opts.unique_md_filenames = unique_md_filenames if opts.compress_type is None: opts.compress_type = compress_type repomd.checksum_type = opts.sumtype repomd.unique_md_filenames = opts.unique_md_filenames repomd.compress = opts.compress if opts.compress_type not in _available_compression: print "Compression %s not available: Please choose from: %s" % (opts.compress_type, ', '.join(_available_compression)) return 1 if opts.sumtype not in _available_checksums: print "Checksum %s not available: Please choose from: %s" % (opts.sumtype, ', '.join(_available_checksums)) return 1 repomd.compress_type = opts.compress_type # remove if opts.remove: try: repomd.remove(metadata, mdtype=opts.mdtype) except MDError, ex: print "Could not remove metadata: %s" % (metadata, str(ex)) return 1 return # add try: repomd.add(metadata, mdtype=opts.mdtype) except MDError, e: print "Could not add metadata from file %s: %s" % (metadata, str(e)) return 1 if __name__ == '__main__': ret = main(sys.argv[1:]) sys.exit(ret) createrepo-0.10.3/createrepo.spec0000664000175000017500000000477112271721200016660 0ustar zpavlaszpavlas%{!?python_sitelib: %define python_sitelib %(python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %if ! 0%{?rhel} # we don't have this in rhel yet... BuildRequires: bash-completion %endif # disable broken /usr/lib/rpm/brp-python-bytecompile %define __os_install_post %{nil} %define compdir %(pkg-config --variable=completionsdir bash-completion) %if "%{compdir}" == "" %define compdir "/etc/bash_completion.d" %endif Summary: Creates a common metadata repository Name: createrepo Version: 0.10.3 Release: 1 License: GPL Group: System Environment/Base Source: %{name}-%{version}.tar.gz URL: http://createrepo.baseurl.org/ BuildRoot: %{_tmppath}/%{name}-%{version}root BuildArchitectures: noarch Requires: python >= 2.1, rpm-python, rpm >= 0:4.1.1, libxml2-python Requires: yum-metadata-parser, yum >= 3.2.29, python-deltarpm, pyliblzma %description This utility will generate a common metadata repository from a directory of rpm packages %prep %setup -q %install [ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT make DESTDIR=$RPM_BUILD_ROOT sysconfdir=%{_sysconfdir} install %clean [ "$RPM_BUILD_ROOT" != "/" ] && rm -rf $RPM_BUILD_ROOT %files %defattr(-, root, root) %dir %{_datadir}/%{name} %doc ChangeLog README COPYING COPYING.lib %(dirname %{compdir}) %{_datadir}/%{name}/* %{_bindir}/%{name} %{_bindir}/modifyrepo %{_bindir}/mergerepo %{_mandir}/man8/createrepo.8* %{_mandir}/man1/modifyrepo.1* %{_mandir}/man1/mergerepo.1* %{python_sitelib}/createrepo %changelog * Fri Sep 9 2011 Seth Vidal - add lzma dep * Wed Jan 26 2011 Seth Vidal - bump to 0.9.9 - add worker.py * Thu Aug 19 2010 Seth Vidal - increase yum requirement for the modifyrepo use of RepoMD, RepoData and RepoMDError * Fri Aug 28 2009 Seth Vidal - 0.9.8 * Tue Mar 24 2009 Seth Vidal - 0.9.7 * Fri Oct 17 2008 Seth Vidal - add mergerepo - 0.9.6 * Mon Feb 18 2008 Seth Vidal - 0.9.5 * Mon Jan 28 2008 Seth Vidal - 0.9.4 * Tue Jan 22 2008 Seth Vidal - 0.9.3 * Thu Jan 17 2008 Seth Vidal - significant api changes * Tue Jan 8 2008 Seth Vidal - 0.9.1 - lots of fixes - cleanup changelog, too * Thu Dec 20 2007 Seth Vidal - beginning of the new version createrepo-0.10.3/dmd.py0000775000175000017500000001167012271721200014770 0ustar zpavlaszpavlas#!/usr/bin/python # dmd - Generate and apply deltas between repository metadata # # Copyright (C) 2007 James Bowes # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. import sys from lxml.etree import parse, tostring, Element class MdType(object): def __init__(self, namespace, rootelem): self.ns = "http://linux.duke.edu/metadata/%s" % namespace self.sns = "{%s}" % self.ns self.deltasns = "{http://linux.duke.edu/metadata/delta}" self.root = rootelem def get_pkg_id(self, pkg): return pkg.findtext(self.sns + "checksum") def make_hash(self, tree): pkgshash = {} for pkg in tree: pkgid = self.get_pkg_id(pkg) pkgshash[pkgid] = pkg return pkgshash def make_pkg_elem(self, pkgid, pkg): pkgelem = Element("package") pkgelem.set('name', pkg.findtext(self.sns + 'name')) pkgelem.set('arch', pkg.findtext(self.sns + 'arch')) pkgelem.set('pkgid', pkgid) verelem = pkg.find(self.sns + 'version') verelem.tag = "version" pkgelem.append(verelem) return pkgelem def diff_trees(self, oldtree, newtree): oldpkgs = oldtree.getroot().getchildren() newpkgs = newtree.getroot().getchildren() oldpkgshash = self.make_hash(oldpkgs) newpkgshash = self.make_hash(newpkgs) diff = Element(self.root, nsmap = {None : self.ns, "rpm" : "http://linux.duke.edu/metadata/rpm", "delta" : "http://linux.duke.edu/metadata/delta"}) additions = Element("delta:additions") diff.append(additions) removals = Element("delta:removals") diff.append(removals) for pkgid, pkg in newpkgshash.iteritems(): if not oldpkgshash.has_key(pkgid): additions.append(pkg) for pkgid, pkg in oldpkgshash.iteritems(): if not newpkgshash.has_key(pkgid): pkgelem = self.make_pkg_elem(pkgid, pkg) removals.append(pkgelem) diff.set("packages", str(len(removals) + len(additions))) print tostring(diff, pretty_print=True) def patch_tree(self, oldtree, deltatree): oldroot = oldtree.getroot() oldpkgs = oldroot.getchildren() oldpkgshash = self.make_hash(oldpkgs) additions = deltatree.find(self.deltasns + 'additions').getchildren() removals = deltatree.find(self.deltasns + 'removals').getchildren() for pkg in additions: pkgid = self.get_pkg_id(pkg) if oldpkgshash.has_key(pkgid): print >> sys.stderr, "Package %s already exists" % pkgid sys.exit(1) oldroot.append(pkg) for pkg in removals: pkgid = pkg.get('pkgid') if not oldpkgshash.has_key(pkgid): print >> sys.stderr, "Package %s does not exist" % pkgid sys.exit(1) oldroot.remove(oldpkgshash[pkgid]) oldcount = int(oldroot.get('packages')) newcount = oldcount + len(additions) - len(removals) oldroot.set('packages', str(newcount)) print tostring(oldtree, pretty_print=True) class OtherMdType(MdType): def get_pkg_id(self, pkg): return pkg.get('pkgid') def make_pkg_elem(self, pkgid, pkg): pkgelem = Element("package") pkgelem.set('name', pkg.get('name')) pkgelem.set('arch', pkg.get('arch')) pkgelem.set('pkgid', pkgid) verelem = pkg.find(self.sns + 'version') verelem.tag = "version" return pkgelem mdtypeinfo = { 'primary' : MdType('common', 'metadata'), 'filelists' : OtherMdType('filelists', 'filelists'), 'other' : OtherMdType('other', 'other'), } def usage(progname): print "usage: %s [diff|patch] MDTYPE FILE1 FILE2" % progname sys.exit() def main(args): if len(args) != 5: usage(args[0]) if args[1] not in ('diff', 'patch'): usage(args[0]) if args[2] not in ('primary', 'filelists', 'other'): usage(args[0]) oldtree = parse(args[3]) newtree = parse(args[4]) if args[1] == 'diff': mdtypeinfo[args[2]].diff_trees(oldtree, newtree) else: mdtypeinfo[args[2]].patch_tree(oldtree, newtree) if __name__ == "__main__": main(sys.argv) createrepo-0.10.3/createrepo.bash0000664000175000017500000001104212271721200016630 0ustar zpavlaszpavlas# bash completion for createrepo and friends _cr_compress_type() { COMPREPLY=( $( compgen -W "$( ${1:-createrepo} --compress-type=FOO / 2>&1 \ | sed -ne 's/,/ /g' -ne 's/.*[Cc]ompression.*://p' )" -- "$2" ) ) } _cr_checksum_type() { COMPREPLY=( $( compgen -W 'md5 sha1 sha256 sha512' -- "$1" ) ) } _cr_createrepo() { COMPREPLY=() case $3 in --version|-h|--help|-u|--baseurl|--distro|--content|--repo|\ --revision|-x|--excludes|--changelog-limit|--max-delta-rpm-size) return 0 ;; --basedir|-c|--cachedir|--update-md-path|-o|--outputdir|\ --oldpackagedirs) local IFS=$'\n' COMPREPLY=( $( compgen -d -- "$2" ) ) return 0 ;; -g|--groupfile) local IFS=$'\n' COMPREPLY=( $( compgen -f -o plusdirs -X '!*.xml' -- "$2" ) ) return 0 ;; -s|--checksum) _cr_checksum_type "$2" return 0 ;; -i|--pkglist|--read-pkgs-list) local IFS=$'\n' COMPREPLY=( $( compgen -f -o plusdirs -- "$2" ) ) return 0 ;; -n|--includepkg) local IFS=$'\n' COMPREPLY=( $( compgen -f -o plusdirs -X '!*.rpm' -- "$2" ) ) return 0 ;; --retain-old-md) COMPREPLY=( $( compgen -W '0 1 2 3 4 5 6 7 8 9' -- "$2" ) ) return 0 ;; --num-deltas) COMPREPLY=( $( compgen -W '1 2 3 4 5 6 7 8 9' -- "$2" ) ) return 0 ;; --workers) local min=2 max=$( getconf _NPROCESSORS_ONLN 2>/dev/null ) [[ -z $max || $max -lt $min ]] && max=$min COMPREPLY=( $( compgen -W "{1..$max}" -- "$2" ) ) return 0 ;; --compress-type) _cr_compress_type "$1" "$2" return 0 ;; esac if [[ $2 == -* ]] ; then COMPREPLY=( $( compgen -W '--version --help --quiet --verbose --profile --excludes --basedir --baseurl --groupfile --checksum --pretty --cachedir --checkts --no-database --update --update-md-path --skip-stat --split --pkglist --includepkg --outputdir --skip-symlinks --changelog-limit --unique-md-filenames --simple-md-filenames --retain-old-md --distro --content --repo --revision --deltas --oldpackagedirs --num-deltas --read-pkgs-list --max-delta-rpm-size --workers --compress-type' -- "$2" ) ) else local IFS=$'\n' COMPREPLY=( $( compgen -d -- "$2" ) ) fi } && complete -F _cr_createrepo -o filenames createrepo genpkgmetadata.py _cr_mergerepo() { COMPREPLY=() case $3 in --version|-h|--help|-a|--archlist) return 0 ;; -r|--repo|-o|--outputdir) local IFS=$'\n' COMPREPLY=( $( compgen -d -- "$2" ) ) return 0 ;; --compress-type) _cr_compress_type "" "$2" return 0 ;; esac COMPREPLY=( $( compgen -W '--version --help --repo --archlist --no-database --outputdir --nogroups --noupdateinfo --compress-type' -- "$2" ) ) } && complete -F _cr_mergerepo -o filenames mergerepo mergerepo.py _cr_modifyrepo() { COMPREPLY=() case $3 in --version|-h|--help|--mdtype) return 0 ;; --compress-type) _cr_compress_type "" "$2" return 0 ;; -s|--checksum) _cr_checksum_type "$2" return 0 ;; esac if [[ $2 == -* ]] ; then COMPREPLY=( $( compgen -W '--version --help --mdtype --remove --compress --no-compress --compress-type --checksum --unique-md-filenames --simple-md-filenames' -- "$2" ) ) return 0 fi local i argnum=1 for (( i=1; i < ${#COMP_WORDS[@]}-1; i++ )) ; do if [[ ${COMP_WORDS[i]} != -* && ${COMP_WORDS[i-1]} != @(=|--@(md|compress-)type) ]]; then argnum=$(( argnum+1 )) fi done local IFS=$'\n' case $argnum in 1) COMPREPLY=( $( compgen -f -o plusdirs -- "$2" ) ) return 0 ;; 2) COMPREPLY=( $( compgen -d -- "$2" ) ) return 0 ;; esac } && complete -F _cr_modifyrepo -o filenames modifyrepo modifyrepo.py # Local variables: # mode: shell-script # sh-basic-offset: 4 # sh-indent-comment: t # indent-tabs-mode: nil # End: # ex: ts=4 sw=4 et filetype=sh createrepo-0.10.3/worker.py0000775000175000017500000000763012271721200015536 0ustar zpavlaszpavlas#!/usr/bin/python -tt import sys import yum import createrepo import os import rpmUtils import re from optparse import OptionParser # pass in dir to make tempdirs in # make tempdir for this worker # create 3 files in that tempdir # return how many pkgs # return on stderr where things went to hell #TODO - take most of read_in_package from createrepo and duplicate it here # so we can do downloads, etc. # then replace callers of read_in_package with forked callers of this # and reassemble at the end def main(args): parser = OptionParser() parser.add_option('--tmpmdpath', default=None, help="path where the outputs should be dumped for this worker") parser.add_option('--pkglist', default=None, help="file to read the pkglist from in lieu of all of them on the cli") parser.add_option("--pkgoptions", default=[], action='append', help="pkgoptions in the format of key=value") parser.add_option("--quiet", default=False, action='store_true', help="only output errors and a total") parser.add_option("--verbose", default=False, action='store_true', help="output errors and a total") parser.add_option("--globalopts", default=[], action='append', help="general options in the format of key=value") opts, pkgs = parser.parse_args(args) external_data = {'_packagenumber': 1} globalopts = {} for strs in opts.pkgoptions: k,v = strs.split('=') if v in ['True', 'true', 'yes', '1', 1]: v = True elif v in ['False', 'false', 'no', '0', 0]: v = False elif v in ['None', 'none', '']: v = None external_data[k] = v for strs in opts.globalopts: k,v = strs.split('=') if v in ['True', 'true', 'yes', '1', 1]: v = True elif v in ['False', 'false', 'no', '0', 0]: v = False elif v in ['None', 'none', '']: v = None globalopts[k] = v # turn off buffering on stdout sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) reldir = external_data['_reldir'] ts = rpmUtils.transaction.initReadOnlyTransaction() if opts.tmpmdpath: files = [open(opts.tmpmdpath + '/%s.xml' % i, 'w') for i in ('primary', 'filelists', 'other')] def output(*xml): for fh, buf in zip(files, xml): fh.write(buf) else: def output(*xml): buf = ' '.join(str(len(i)) for i in xml) sys.stdout.write('*** %s\n' % buf) for buf in xml: sys.stdout.write(buf) if opts.pkglist: for line in open(opts.pkglist,'r').readlines(): line = line.strip() if re.match('^\s*\#.*', line) or re.match('^\s*$', line): continue pkgs.append(line) clog_limit=globalopts.get('clog_limit', None) if clog_limit is not None: clog_limit = int(clog_limit) for pkgfile in pkgs: pkgpath = reldir + '/' + pkgfile if not os.path.exists(pkgpath): print >> sys.stderr, "File not found: %s" % pkgpath output() continue try: if not opts.quiet and opts.verbose: print "reading %s" % (pkgfile) pkg = createrepo.yumbased.CreateRepoPackage(ts, package=pkgpath, sumtype=globalopts.get('sumtype', None), external_data=external_data) output(pkg.xml_dump_primary_metadata(), pkg.xml_dump_filelists_metadata(), pkg.xml_dump_other_metadata(clog_limit=clog_limit)) except yum.Errors.YumBaseError, e: print >> sys.stderr, "Error: %s" % e output() continue else: external_data['_packagenumber']+=1 if __name__ == "__main__": main(sys.argv[1:]) createrepo-0.10.3/bin/0000775000175000017500000000000012271721201014413 5ustar zpavlaszpavlascreaterepo-0.10.3/bin/mergerepo0000775000175000017500000000006712271721201016331 0ustar zpavlaszpavlas#!/bin/sh exec /usr/share/createrepo/mergerepo.py "$@" createrepo-0.10.3/bin/createrepo0000775000175000017500000000007412271721201016473 0ustar zpavlaszpavlas#!/bin/sh exec /usr/share/createrepo/genpkgmetadata.py "$@" createrepo-0.10.3/bin/Makefile0000664000175000017500000000366612271721201016066 0ustar zpavlaszpavlasSHELL = /bin/sh top_srcdir = .. srcdir = ../bin prefix = /usr exec_prefix = ${prefix} bindir = ${exec_prefix}/bin sbindir = ${exec_prefix}/sbin libexecdir = ${exec_prefix}/libexec datadir = ${prefix}/share sysconfdir = ${prefix}/etc sharedstatedir = ${prefix}/com localstatedir = ${prefix}/var libdir = ${exec_prefix}/lib infodir = ${prefix}/info docdir = includedir = ${prefix}/include oldincludedir = /usr/include mandir = ${prefix}/man pkgdatadir = $(datadir)/$(PKGNAME) pkglibdir = $(libdir)/$(PKGNAME) pkgincludedir = $(includedir)/$(PKGNAME) top_builddir = ../ # all dirs DIRS = $(DESTDIR)$(bindir) $(DESTDIR)/etc $(DESTDIR)$(pkgdatadir) # INSTALL scripts INSTALL = install -p --verbose INSTALL_BIN = $(INSTALL) -m 755 INSTALL_DIR = $(INSTALL) -m 755 -d INSTALL_DATA = $(INSTALL) -m 644 INSTALL_MODULES = $(INSTALL) -m 755 -D RM = rm -f all: echo "Nothing to do" install: all installdirs $(INSTALL_BIN) $(srcdir)/$(PKGNAME) $(DESTDIR)$(bindir)/$(PKGNAME) $(INSTALL_BIN) $(srcdir)/modifyrepo $(DESTDIR)$(bindir)/modifyrepo $(INSTALL_BIN) $(srcdir)/mergerepo $(DESTDIR)$(bindir)/mergerepo uninstall: $(RM) $(bindir)/$(PKGNAME) clean: distclean: $(RM) -rf .libs $(RM) -f core $(RM) -f *~ mostlyclean: $(MAKE) clean maintainer-clean: $(MAKE) distclean distfiles: distdir=$(PKGNAME)-$(VERSION); \ mkdir $(top_srcdir)/.disttmp/$$distdir/bin;\ cp \ $(srcdir)/$(PKGNAME) \ $(srcdir)/Makefile \ $(srcdir)/modifyrepo \ $(srcdir)/mergerepo \ $(top_srcdir)/.disttmp/$$distdir/bin dailyfiles: distdir=$(PKGNAME); \ mkdir $(top_srcdir)/.disttmp/$$distdir/bin;\ cp \ $(srcdir)/$(PKGNAME) \ $(srcdir)/Makefile \ $(srcdir)/modifyrepo \ $(srcdir)/mergerepo \ $(top_srcdir)/.disttmp/$$distdir/bin installdirs: $(MAKE) -C $(top_srcdir) installdirs .PHONY: all install install-strip uninstall clean distclean mostlyclean maintainer-clean info dvi dist distfiles check installcheck installdirs dailyfiles createrepo-0.10.3/bin/modifyrepo0000775000175000017500000000007012271721201016513 0ustar zpavlaszpavlas#!/bin/sh exec /usr/share/createrepo/modifyrepo.py "$@" createrepo-0.10.3/docs/0000775000175000017500000000000012271721201014573 5ustar zpavlaszpavlascreaterepo-0.10.3/docs/modifyrepo.10000664000175000017500000000175412271721201017041 0ustar zpavlaszpavlas.TH "modifyrepo" "1" "2007 Dec 3" "Luke Macken" "" .SH "NAME" modifyrepo \- Modify a repomd (xml-rpm-metadata) repository .SH "SYNOPSIS" \fBmodifyrepo\fP [options] .PP .SH "DESCRIPTION" \fBmodifyrepo\fP is a program that allows you to insert arbitrary metadata into a repomd (xml-based rpm metadata) repository. .SH "EXAMPLES" .PP $ \fBmodifyrepo\fP \-\-mdtype=newmd metadata.xml /repository/repodata .br Wrote: /repository/repodata/metadata.xml.gz type = newmd location = repodata/metadata.xml.gz checksum = 1d7ee93db2964e7f85e07ec19b3204591da1050c timestamp = 1196716296.0 open-checksum = 824d936dc7dfff029379797b311af0cc66af4115 .br Wrote: /repository/repodata/repomd.xml .PP .SH "SEE ALSO" .I createrepo (8) .PP .SH "AUTHORS" .nf Luke Macken Seth Vidal .fi .PP .SH "BUGS" Any bugs which are found should be emailed to the mailing list: rpm-metadata@lists.baseurl.org .fi createrepo-0.10.3/docs/Makefile0000664000175000017500000000404112271721201016232 0ustar zpavlaszpavlasSHELL = /bin/sh top_srcdir = .. srcdir = ../docs prefix = /usr exec_prefix = ${prefix} bindir = ${exec_prefix}/bin sbindir = ${exec_prefix}/sbin libexecdir = ${exec_prefix}/libexec datadir = ${prefix}/share sysconfdir = ${prefix}/etc sharedstatedir = ${prefix}/com localstatedir = ${prefix}/var libdir = ${exec_prefix}/lib infodir = ${prefix}/info docdir = includedir = ${prefix}/include oldincludedir = /usr/include mandir = ${datadir}/man pkgdatadir = $(datadir)/$(PKGNAME) pkglibdir = $(libdir)/$(PKGNAME) pkgincludedir = $(includedir)/$(PKGNAME) top_builddir = ../ # all dirs DIRS = $(DESTDIR)$(bindir) $(DESTDIR)/etc $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(mandir) # INSTALL scripts INSTALL = install -p --verbose INSTALL_BIN = $(INSTALL) -m 755 INSTALL_DIR = $(INSTALL) -m 755 -d INSTALL_DATA = $(INSTALL) -m 644 INSTALL_MODULES = $(INSTALL) -m 755 -D RM = rm -f all: echo "nothing to do" install: all installdirs mkdir -p $(DESTDIR)$(mandir)/man8 mkdir -p $(DESTDIR)$(mandir)/man1 $(INSTALL_DATA) createrepo.8 $(DESTDIR)$(mandir)/man8/createrepo.8 $(INSTALL_DATA) modifyrepo.1 $(DESTDIR)$(mandir)/man1/modifyrepo.1 $(INSTALL_DATA) mergerepo.1 $(DESTDIR)$(mandir)/man1/mergerepo.1 uninstall: $(RM) $(bindir)/$(PKGNAME) clean: distclean: $(RM) -rf .libs $(RM) -f core $(RM) -f *~ mostlyclean: $(MAKE) clean maintainer-clean: $(MAKE) distclean distfiles: distdir=$(PKGNAME)-$(VERSION); \ mkdir $(top_srcdir)/.disttmp/$$distdir/docs;\ cp \ $(srcdir)/createrepo.8 \ $(srcdir)/modifyrepo.1 \ $(srcdir)/mergerepo.1 \ $(srcdir)/Makefile \ $(top_srcdir)/.disttmp/$$distdir/docs dailyfiles: distdir=$(PKGNAME); \ mkdir $(top_srcdir)/.disttmp/$$distdir/docs;\ cp \ $(srcdir)/createrepo.8 \ $(srcdir)/modifyrepo.1 \ $(srcdir)/mergerepo.1 \ $(srcdir)/Makefile \ $(top_srcdir)/.disttmp/$$distdir/docs installdirs: $(MAKE) -C $(top_srcdir) installdirs .PHONY: all install install-strip uninstall clean distclean mostlyclean maintainer-clean info dvi dist distfiles check installcheck installdirs dailyfiles createrepo-0.10.3/docs/mergerepo.10000664000175000017500000000255212271721201016646 0ustar zpavlaszpavlas.TH "mergerepo" "1" "2008 Oct 21" "Seth Vidal" "" .SH "NAME" mergerepo \- Merge multiple repositories together .SH "SYNOPSIS" \fBmergerepo\fP \-\-repo repo1 \-\-repo repo2 .PP .SH "DESCRIPTION" \fBmergerepo\fP is a program that allows you merge multiple repositories into a single repository while referring to the remote location for all packages. .SH "OPTIONS" .IP "\fB\-r \-\-repo\fP " Url to a repository to be merged. .IP "\fB\-o \-\-outputdir \fP" Path where merged repository metadata should be written to. If not specified repository metadata will be written to `pwd`/merged_repo/. .IP "\fB\-d \-\-database\fP" Generate sqlite databases of the merged repository metadata. .IP "\fB\-a \-\-archlist\fP" Specify a comma-separated list of architectures to use. Defaults to ALL. .IP "\fB\-\-nogroups\fP" Do not merge/include groups metadata in the repository. .IP "\fB\-\-noupdateinfo\fP" Do not merge/include updateinfo metadata in the repository. .SH "EXAMPLES" .PP $ \fBmergerepo\fP \-\-repo=http://myurl.org/repo1 \-\-repo=http://myurl.org/repo2 \-d \-o /tmp/mymergedrepo .PP .SH "SEE ALSO" .I createrepo (8) .PP .SH "AUTHORS" .nf Seth Vidal .fi .PP .SH "BUGS" Any bugs which are found should be emailed to the mailing list: rpm-metadata@lists.baseurl.org or filed as tickets at: http://createrepo.baseurl.org/ .fi createrepo-0.10.3/docs/createrepo.80000664000175000017500000001340512271721201017020 0ustar zpavlaszpavlas.TH "createrepo" "8" "2005 Jan 2" "Seth Vidal" "" .SH "NAME" createrepo \- Create repomd (xml-rpm-metadata) repository .SH "SYNOPSIS" \fBcreaterepo\fP [options] .PP .SH "DESCRIPTION" \fBcreaterepo\fP is a program that creates a repomd (xml-based rpm metadata) repository from a set of rpms. .SH "OPTIONS" .IP "\fB\-u \-\-baseurl\fP " Optional base URL location for all files. .IP "\fB\-o \-\-outputdir\fP " Optional output directory (useful for read only media). .IP "\fB\-x \-\-excludes\fP " File globs to exclude, can be specified multiple times. .IP "\fB\-i \-\-pkglist\fP " specify a text file which contains the complete list of files to include in the repository from the set found in the directory. File format is one package per line, no wildcards or globs. .IP "\fB\-n \-\-includepkg\fP" specify pkgs to include on the command line. Takes urls as well as local paths. .IP "\fB\-q \-\-quiet\fP" Run quietly. .IP "\fB\-g \-\-groupfile\fP " A precreated xml filename to point to for group information. .br See examples section below for further explanation. .IP "\fB\-v \-\-verbose\fP" Run verbosely. .IP "\fB\-c \-\-cachedir\fP " Specify a directory to use as a cachedir. This allows createrepo to create a cache of checksums of packages in the repository. In consecutive runs of createrepo over the same repository of files that do not have a complete change out of all packages this decreases the processing time dramatically. .br .IP "\fB\-\-basedir\fP" Basedir for path to directories in the repodata, default is the current working directory. .br .IP "\fB\-\-update\fP" If metadata already exists in the outputdir and an rpm is unchanged (based on file size and mtime) since the metadata was generated, reuse the existing metadata rather than recalculating it. In the case of a large repository with only a few new or modified rpms this can significantly reduce I/O and processing time. .br .IP "\fB\-\-skip\-stat\fP" skip the stat() call on a \-\-update, assumes if the filename is the same then the file is still the same (only use this if you're fairly trusting or gullible). .br .IP "\fB\-\-update\-md\-path\fP" Use the existing repodata for \-\-update, from this path. .br .IP "\fB\-C \-\-checkts\fP" Don't generate repo metadata, if their timestamps are newer than its rpms. This option decreases the processing time drastically again, if you happen to run it on an unmodified repo, but it is (currently) mutual exclusive with the \-\-split option. NOTE: This command will not notice when packages have been removed from repo. Use \-\-update to handle that. .br .IP "\fB\-\-split\fP" Run in split media mode. Rather than pass a single directory, take a set of directories corresponding to different volumes in a media set. .br .IP "\fB\-p \-\-pretty\fP" Output xml files in pretty format. .IP "\fB\-\-version\fP" Output version. .IP "\fB\-h \-\-help\fP" Show help menu. .IP "\fB\-d \-\-database\fP" Generate sqlite databases for use with yum. This is now the default. .IP "\fB\-\-no\-database\fP" Do not generate sqlite databases in the repository. .IP "\fB\-S \-\-skip\-symlinks\fP" Ignore symlinks of packages .IP "\fB\-s \-\-checksum\fP" Choose the checksum type used in repomd.xml and for packages in the metadata. The default is now "sha256" (if python has hashlib). The older default was "sha", which is actually "sha1", however explicitly using "sha1" doesn't work on older (3.0.x) versions of yum, you need to specify "sha". .IP "\fB\-\-profile\fP" Output time based profiling information. .IP "\fB\-\-changelog\-limit\fP CHANGELOG_LIMIT" Only import the last N changelog entries, from each rpm, into the metadata .IP "\fB\-\-unique\-md\-filenames\fP" Include the file's checksum in the metadata filename, helps HTTP caching (default) .IP "\fB\-\-simple\-md\-filenames\fP" Do not include the file's checksum in the metadata filename. .IP "\fB\-\-retain\-old\-md\fP" Keep around the latest (by timestamp) N copies of the old repodata (so clients with older repomd.xml files can still access it). Default is 0. .IP "\fB\-\-distro\fP" Specify distro tags. Can be specified more than once. Optional syntax specifying a cpeid(http://cpe.mitre.org/) \-\-distro=cpeid,distrotag .IP "\fB\-\-content\fP" Specify keyword/tags about the content of the repository. Can be specified more than once. .IP "\fB\-\-repo\fP" Specify keyword/tags about the repository itself. Can be specified more than once. .IP "\fB\-\-revision\fP" Arbitrary string for a repository revision. .IP "\fB\-\-deltas\fP" Tells createrepo to generate deltarpms and the delta metadata .IP "\fB\-\-oldpackagedirs\fP PATH" paths to look for older pkgs to delta against. Can be specified multiple times .IP "\fB\-\-num\-deltas\fP int" the number of older versions to make deltas against. Defaults to 1 .IP "\fB\-\-read\-pkgs\-list\fP READ_PKGS_LIST output the paths to the pkgs actually read useful with \-\-update .IP "\fB\-\-max\-delta\-rpm\-size\fP MAX_DELTA_RPM_SIZE max size of an rpm that to run deltarpm against (in bytes) .IP "\fB\-\-workers\fP WORKERS number of workers to spawn to read rpms .IP "\fB\-\-compress\-type\fP specify which compression method to use: compat (default), xz (may not be available), gz, bz2. .IP .SH "EXAMPLES" Here is an example of a repository with a groups file. Note that the groups file should be in the same directory as the rpm packages (i.e. /path/to/rpms/comps.xml). .br .PP \fBcreaterepo\fP \-g comps.xml /path/to/rpms .SH "FILES" .nf repodata/filelists.xml.gz repodata/other.xml.gz repodata/primary.xml.gz repodata/repomd.xml .fi .PP .SH "SEE ALSO" .I yum (8) yum.conf (5) .PP .SH "AUTHORS" .nf See the Authors file .fi .PP .SH "BUGS" Any bugs which are found should be emailed to the mailing list: rpm-metadata@lists.baseurl.org or reported in trac at: http://createrepo.baseurl.org .fi createrepo-0.10.3/createrepo/0000775000175000017500000000000012271721201015774 5ustar zpavlaszpavlascreaterepo-0.10.3/createrepo/merge.py0000664000175000017500000001274212271721201017453 0ustar zpavlaszpavlas#!/usr/bin/python -tt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2008 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org # merge repos from arbitrary repo urls import os import shutil import yum import yum.Errors from yum.misc import unique, getCacheDir import yum.update_md import rpmUtils.arch import operator from utils import MDError import createrepo import tempfile # take repo paths from cli # produce new repo metadata from merging the two together. #TODO: # excludes? class RepoMergeBase: def __init__(self, repolist=[], yumbase=None, mdconf=None, mdbase_class=None ): self.repolist = repolist self.outputdir = '%s/merged_repo' % os.getcwd() self.exclude_tuples = [] self.sort_func = self._sort_func # callback function to magically sort pkgs if not mdconf: self.mdconf = createrepo.MetaDataConfig() else: self.mdconf = mdconf if not mdbase_class: self.mdbase_class = createrepo.MetaDataGenerator else: self.mdbase_class = mdbase_class if not yumbase: self.yumbase = yum.YumBase() else: self.yumbase = yumbase self.yumbase.conf.cachedir = getCacheDir() self.yumbase.conf.cache = 0 # default to all arches self.archlist = unique(rpmUtils.arch.arches.keys() + rpmUtils.arch.arches.values()) self.groups = True self.updateinfo = True def _sort_func(self, repos): """Default sort func for repomerge. Takes a list of repository objects any package which is not to be included in the merged repo should be delPackage()'d""" # sort the repos by _merge_rank # - lowest number is the highest rank (1st place, 2ndplace, etc) repos.sort(key=operator.attrgetter('_merge_rank')) for repo in repos: for pkg in repo.sack: others = self.yumbase.pkgSack.searchNevra(pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch) if len(others) > 1: for thatpkg in others: if pkg.repoid == thatpkg.repoid: continue if pkg.repo._merge_rank < thatpkg.repo._merge_rank: thatpkg.repo.sack.delPackage(thatpkg) def merge_repos(self): self.yumbase.repos.disableRepo('*') # add our repos and give them a merge rank in the order they appear in # in the repolist count = 0 for r in self.repolist: if ':' not in r: r = os.path.abspath(r) r = 'file://' + r # just fix the file repos, this is silly. count +=1 rid = 'repo%s' % count n = self.yumbase.add_enable_repo(rid, baseurls=[r], metadata_expire=0, timestamp_check=False) n._merge_rank = count #setup our sacks try: self.yumbase._getSacks(archlist=self.archlist) except yum.Errors.RepoError, e: raise MDError, "Could not setup merge repo pkgsack: %s" % e myrepos = self.yumbase.repos.listEnabled() self.sort_func(myrepos) def write_metadata(self, outputdir=None): mytempdir = tempfile.mkdtemp() if self.groups: try: comps_fn = mytempdir + '/groups.xml' compsfile = open(comps_fn, 'w') compsfile.write(self.yumbase.comps.xml()) compsfile.close() except yum.Errors.GroupsError, e: # groups not being available shouldn't be a fatal error pass else: self.mdconf.groupfile=comps_fn if self.updateinfo: ui_fn = mytempdir + '/updateinfo.xml' uifile = open(ui_fn, 'w') umd = yum.update_md.UpdateMetadata() for repo in self.yumbase.repos.listEnabled(): try: # attempt to grab the updateinfo.xml.gz from the repodata umd.add(repo) except yum.Errors.RepoMDError: continue umd.xml(fileobj=uifile) uifile.close() self.mdconf.additional_metadata['updateinfo'] = ui_fn self.mdconf.pkglist = self.yumbase.pkgSack self.mdconf.directory = self.outputdir if outputdir: self.mdconf.directory = outputdir # clean out what was there if os.path.exists(self.mdconf.directory + '/repodata'): shutil.rmtree(self.mdconf.directory + '/repodata') if not os.path.exists(self.mdconf.directory): os.makedirs(self.mdconf.directory) mdgen = self.mdbase_class(config_obj=self.mdconf) mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove() createrepo-0.10.3/createrepo/yumbased.py0000664000175000017500000002215312271721200020161 0ustar zpavlaszpavlas#!/usr/bin/python -tt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2007 Red Hat, Inc - written by seth vidal skvidal at fedoraproject.org import os def _get_umask(): oumask = os.umask(0) os.umask(oumask) return oumask _b4rpm_oumask = _get_umask() import rpm import types from yum.packages import YumLocalPackage from yum.Errors import * from yum import misc import utils import tempfile class CreateRepoPackage(YumLocalPackage): def __init__(self, ts, package, sumtype=None, external_data={}): YumLocalPackage.__init__(self, ts, package) if sumtype: self.checksum_type = sumtype if external_data: for (key, val) in external_data.items(): setattr(self, key, val) def _do_checksum(self): """return a checksum for a package: - check if the checksum cache is enabled if not - return the checksum if so - check to see if it has a cache file if so, open it and return the first line's contents if not, grab the checksum and write it to a file for this pkg """ # already got it if self._checksum: return self._checksum # not using the cachedir if not hasattr(self, '_cachedir') or not self._cachedir: self._checksum = misc.checksum(self.checksum_type, self.localpath) self._checksums = [(self.checksum_type, self._checksum, 1)] return self._checksum t = [] if type(self.hdr[rpm.RPMTAG_SIGGPG]) is not types.NoneType: t.append("".join(self.hdr[rpm.RPMTAG_SIGGPG])) if type(self.hdr[rpm.RPMTAG_SIGPGP]) is not types.NoneType: t.append("".join(self.hdr[rpm.RPMTAG_SIGPGP])) if type(self.hdr[rpm.RPMTAG_HDRID]) is not types.NoneType: t.append("".join(self.hdr[rpm.RPMTAG_HDRID])) kcsum = misc.Checksums(checksums=[self.checksum_type]) kcsum.update("".join(t)) key = kcsum.hexdigest() csumtag = '%s-%s-%s-%s' % (os.path.basename(self.localpath), key, self.size, self.filetime) csumfile = '%s/%s' % (self._cachedir, csumtag) if os.path.exists(csumfile) and float(self.filetime) <= float(os.stat(csumfile)[-2]): csumo = open(csumfile, 'r') checksum = csumo.readline() csumo.close() else: checksum = misc.checksum(self.checksum_type, self.localpath) # This is atomic cache creation via. rename, so we can have two # tasks using the same cachedir ... mash does this. try: (csumo, tmpfilename) = tempfile.mkstemp(dir=self._cachedir) csumo = os.fdopen(csumo, 'w', -1) csumo.write(checksum) csumo.close() # tempfile forces 002 ... we want to undo that, so that users # can share the cache. BZ 833350. os.chmod(tmpfilename, 0666 ^ _b4rpm_oumask) os.rename(tmpfilename, csumfile) except: pass self._checksum = checksum self._checksums = [(self.checksum_type, checksum, 1)] return self._checksum # sqlite-direct dump code below here :-/ def _sqlite_null(self, item): if not item: return None return item def do_primary_sqlite_dump(self, cur): """insert primary data in place, this assumes the tables exist""" if self.crp_reldir and self.localpath.startswith(self.crp_reldir): relpath = self.localpath.replace(self.crp_reldir, '') if relpath[0] == '/': relpath = relpath[1:] else: relpath = self.localpath p = (self.crp_packagenumber, self.checksum, self.name, self.arch, self.version, self.epoch, self.release, self.summary.strip(), self.description.strip(), self._sqlite_null(self.url), self.filetime, self.buildtime, self._sqlite_null(self.license), self._sqlite_null(self.vendor), self._sqlite_null(self.group), self._sqlite_null(self.buildhost), self._sqlite_null(self.sourcerpm), self.hdrstart, self.hdrend, self._sqlite_null(self.packager), self.packagesize, self.size, self.archivesize, relpath, self.crp_baseurl, self.checksum_type) q = """insert into packages values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?)""" # write out all of do_primary_sqlite as an executescript - work on the # quoting for pretty much any contingency - take from sqlutils.py # # e #p = None #q = """insert into packages values (%s, %s, %s, %s, """ cur.execute(q, p) # provides, obsoletes, conflicts for pco in ('obsoletes', 'provides', 'conflicts'): thispco = [] for (name, flag, (epoch, ver, rel)) in getattr(self, pco): thispco.append((name, flag, epoch, ver, rel, self.crp_packagenumber)) q = "insert into %s values (?, ?, ?, ?, ?, ?)" % pco cur.executemany(q, thispco) # requires reqs = [] for (name, flag, (epoch, ver, rel), pre) in self._requires_with_pre(): if name.startswith('rpmlib('): continue pre_bool = 'FALSE' if pre == 1: pre_bool = 'TRUE' reqs.append((name, flag, epoch, ver,rel, self.crp_packagenumber, pre_bool)) q = "insert into requires values (?, ?, ?, ?, ?, ?, ?)" cur.executemany(q, reqs) # files p = [] for f in self._return_primary_files(): p.append((f,)) if p: q = "insert into files values (?, 'file', %s)" % self.crp_packagenumber cur.executemany(q, p) # dirs p = [] for f in self._return_primary_dirs(): p.append((f,)) if p: q = "insert into files values (?, 'dir', %s)" % self.crp_packagenumber cur.executemany(q, p) # ghosts p = [] for f in self._return_primary_files(list_of_files = self.returnFileEntries('ghost')): p.append((f,)) if p: q = "insert into files values (?, 'ghost', %s)" % self.crp_packagenumber cur.executemany(q, p) def do_filelists_sqlite_dump(self, cur): """inserts filelists data in place, this assumes the tables exist""" # insert packagenumber + checksum into 'packages' table q = 'insert into packages values (?, ?)' p = (self.crp_packagenumber, self.checksum) cur.execute(q, p) # break up filelists and encode them dirs = {} for (filetype, files) in [('file', self.filelist), ('dir', self.dirlist), ('ghost', self.ghostlist)]: for filename in files: (dirname,filename) = (os.path.split(filename)) if not dirs.has_key(dirname): dirs[dirname] = {'files':[], 'types':[]} dirs[dirname]['files'].append(filename) dirs[dirname]['types'].append(filetype) # insert packagenumber|dir|files|types into files table p = [] for (dirname,direc) in dirs.items(): p.append((self.crp_packagenumber, dirname, utils.encodefilenamelist(direc['files']), utils.encodefiletypelist(direc['types']))) if p: q = 'insert into filelist values (?, ?, ?, ?)' cur.executemany(q, p) def do_other_sqlite_dump(self, cur): """inserts changelog data in place, this assumes the tables exist""" # insert packagenumber + checksum into 'packages' table q = 'insert into packages values (?, ?)' p = (self.crp_packagenumber, self.checksum) cur.execute(q, p) if self.changelog: q = 'insert into changelog ("pkgKey", "date", "author", "changelog") values (%s, ?, ?, ?)' % self.crp_packagenumber cur.executemany(q, self.changelog) def do_sqlite_dump(self, md_sqlite): """write the metadata out to the sqlite dbs""" self.do_primary_sqlite_dump(md_sqlite.primary_cursor) md_sqlite.pri_cx.commit() self.do_filelists_sqlite_dump(md_sqlite.filelists_cursor) md_sqlite.file_cx.commit() self.do_other_sqlite_dump(md_sqlite.other_cursor) md_sqlite.other_cx.commit() createrepo-0.10.3/createrepo/utils.py0000664000175000017500000001422512271721201017512 0ustar zpavlaszpavlas#!/usr/bin/python # util functions for createrepo # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import os import os.path import sys import bz2 import gzip from gzip import write32u, FNAME from yum import misc _available_compression = ['gz', 'bz2'] try: import lzma _available_compression.append('xz') except ImportError: lzma = None def errorprint(stuff): print >> sys.stderr, stuff def _(args): """Stub function for translation""" return args class GzipFile(gzip.GzipFile): def _write_gzip_header(self): # Generate a header that is easily reproduced with gzip -9 -n on # an unix-like system self.fileobj.write('\037\213') # magic header self.fileobj.write('\010') # compression method self.fileobj.write('\000') # flags write32u(self.fileobj, long(0)) # timestamp self.fileobj.write('\002') # max compression self.fileobj.write('\003') # UNIX def _gzipOpen(filename, mode="rb", compresslevel=9): return GzipFile(filename, mode, compresslevel) def bzipFile(source, dest): s_fn = open(source, 'rb') destination = bz2.BZ2File(dest, 'w', compresslevel=9) while True: data = s_fn.read(1024000) if not data: break destination.write(data) destination.close() s_fn.close() def xzFile(source, dest): if not 'xz' in _available_compression: raise MDError, "Cannot use xz for compression, library/module is not available" s_fn = open(source, 'rb') destination = lzma.LZMAFile(dest, 'w') while True: data = s_fn.read(1024000) if not data: break destination.write(data) destination.close() s_fn.close() def gzFile(source, dest): s_fn = open(source, 'rb') destination = GzipFile(dest, 'w') while True: data = s_fn.read(1024000) if not data: break destination.write(data) destination.close() s_fn.close() class Duck: def __init__(self, **attr): self.__dict__ = attr def compressFile(source, dest, compress_type): """Compress an existing file using any compression type from source to dest""" if compress_type == 'xz': xzFile(source, dest) elif compress_type == 'bz2': bzipFile(source, dest) elif compress_type == 'gz': gzFile(source, dest) else: raise MDError, "Unknown compression type %s" % compress_type def compressOpen(fn, mode='rb', compress_type=None): if not compress_type: # we are readonly and we don't give a compress_type - then guess based on the file extension compress_type = fn.split('.')[-1] if compress_type not in _available_compression: compress_type = 'gz' if compress_type == 'xz': fh = lzma.LZMAFile(fn, mode) if mode == 'w': fh = Duck(write=lambda s, write=fh.write: s != '' and write(s), close=fh.close) return fh elif compress_type == 'bz2': return bz2.BZ2File(fn, mode) elif compress_type == 'gz': return _gzipOpen(fn, mode) else: raise MDError, "Unknown compression type %s" % compress_type def returnFD(filename): try: fdno = os.open(filename, os.O_RDONLY) except OSError: raise MDError, "Error opening file" return fdno def checkAndMakeDir(directory): """ check out the directory and make it, if possible, return 1 if done, else return 0 """ if os.path.exists(directory): if not os.path.isdir(directory): #errorprint(_('%s is not a dir') % directory) result = False else: if not os.access(directory, os.W_OK): #errorprint(_('%s is not writable') % directory) result = False else: result = True else: try: os.mkdir(directory) except OSError, e: #errorprint(_('Error creating dir %s: %s') % (directory, e)) result = False else: result = True return result def checksum_and_rename(fn_path, sumtype='sha256'): """checksum the file rename the file to contain the checksum as a prefix return the new filename""" csum = misc.checksum(sumtype, fn_path) fn = os.path.basename(fn_path) fndir = os.path.dirname(fn_path) csum_fn = csum + '-' + fn csum_path = os.path.join(fndir, csum_fn) os.rename(fn_path, csum_path) return (csum, csum_path) def encodefilenamelist(filenamelist): return '/'.join(filenamelist) def encodefiletypelist(filetypelist): result = '' ftl = {'file':'f', 'dir':'d', 'ghost':'g'} for x in filetypelist: result += ftl[x] return result def split_list_into_equal_chunks(seq, num_chunks): """it's used on sorted input which is then merged in order""" out = [[] for i in range(num_chunks)] for i, item in enumerate(seq): out[i % num_chunks].append(item) return out def num_cpus_online(unknown=1): if not hasattr(os, "sysconf"): return unknown if not os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"): return unknown ncpus = os.sysconf("SC_NPROCESSORS_ONLN") try: if int(ncpus) > 0: return ncpus except: pass return unknown class MDError(Exception): def __init__(self, value=None): Exception.__init__(self) self.value = value def __str__(self): return self.value createrepo-0.10.3/createrepo/readMetadata.py0000664000175000017500000001101512271721201020720 0ustar zpavlaszpavlas#!/usr/bin/python -t # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2006 Red Hat import os import stat from utils import errorprint, _ import yum from yum import misc from yum.Errors import YumBaseError import tempfile class CreaterepoPkgOld(yum.sqlitesack.YumAvailablePackageSqlite): # special for special people like us. def _return_remote_location(self): if self.basepath: msg = """\n""" % ( misc.to_xml(self.basepath, attrib=True), misc.to_xml(self.relativepath, attrib=True)) else: msg = """\n""" % misc.to_xml(self.relativepath, attrib=True) return msg class MetadataIndex(object): def __init__(self, outputdir, opts=None): if opts is None: opts = {} self.opts = opts self.outputdir = outputdir realpath = os.path.realpath(outputdir) repodatadir = self.outputdir + '/repodata' self._repo = yum.yumRepo.YumRepository('garbageid') self._repo.baseurl = 'file://' + realpath self._repo.basecachedir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo") self._repo.base_persistdir = tempfile.mkdtemp(dir='/var/tmp', prefix="createrepo-p") self._repo.metadata_expire = 1 self._repo.gpgcheck = 0 self._repo.repo_gpgcheck = 0 self._repo._sack = yum.sqlitesack.YumSqlitePackageSack(CreaterepoPkgOld) self.pkg_tups_by_path = {} try: self.scan() except YumBaseError, e: print "Could not find valid repo at: %s" % self.outputdir def scan(self): """Read in old repodata""" if self.opts.get('verbose'): print _("Scanning old repo data") self._repo.sack.populate(self._repo, 'all', None, False) for thispo in self._repo.sack: if thispo.checksum_type != self.opts['sumtype']: continue mtime = thispo.filetime size = thispo.size relpath = thispo.relativepath do_stat = self.opts.get('do_stat', True) if mtime is None: print _("mtime missing for %s") % relpath continue if size is None: print _("size missing for %s") % relpath continue if do_stat: filepath = os.path.join(self.opts['pkgdir'], relpath) try: st = os.stat(filepath) except OSError: #file missing -- ignore continue if not stat.S_ISREG(st.st_mode): #ignore non files continue #check size and mtime if st.st_size != size: if self.opts.get('verbose'): print _("Size (%i -> %i) changed for file %s") % (size,st.st_size,filepath) continue if int(st.st_mtime) != mtime: if self.opts.get('verbose'): print _("Modification time changed for %s") % filepath continue self.pkg_tups_by_path[relpath] = thispo.pkgtup def getNodes(self, relpath): """return a package object based on relative path of pkg """ if relpath in self.pkg_tups_by_path: pkgtup = self.pkg_tups_by_path[relpath] return self._repo.sack.searchPkgTuple(pkgtup)[0] return None if __name__ == "__main__": cwd = os.getcwd() opts = {'verbose':1, 'pkgdir': cwd} idx = MetadataIndex(cwd, opts) for fn in idx.pkg_tups_by_path: po = idx.getNodes(fn) print po.xml_dump_primary_metadata() print po.xml_dump_filelists_metadata() print po.xml_dump_other_metadata() createrepo-0.10.3/createrepo/__init__.py0000664000175000017500000016537012271721201020121 0ustar zpavlaszpavlas# This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2009 Red Hat, Inc - # written by seth vidal skvidal at fedoraproject.org import os import sys import fnmatch import time import yumbased import shutil from bz2 import BZ2File from urlgrabber import grabber import tempfile import stat import fcntl import subprocess from select import select from yum import misc, Errors from yum.repoMDObject import RepoMD, RepoData from yum.sqlutils import executeSQL from yum.packageSack import MetaSack from yum.packages import YumAvailablePackage import rpmUtils.transaction from utils import _, errorprint, MDError, lzma, _available_compression import readMetadata try: import sqlite3 as sqlite except ImportError: import sqlite try: import sqlitecachec except ImportError: pass from utils import _gzipOpen, compressFile, compressOpen, checkAndMakeDir, GzipFile, \ checksum_and_rename, split_list_into_equal_chunks from utils import num_cpus_online import deltarpms __version__ = '0.9.9' class MetaDataConfig(object): def __init__(self): self.quiet = False self.verbose = False self.profile = False self.excludes = [] self.baseurl = None self.groupfile = None self.sumtype = 'sha256' self.pretty = False self.cachedir = None self.use_cache = False self.basedir = os.getcwd() self.checkts = False self.split = False self.update = False self.deltas = False # do the deltarpm thing # where to put the .drpms - defaults to 'drpms' inside 'repodata' self.deltadir = None self.delta_relative = 'drpms/' self.oldpackage_paths = [] # where to look for the old packages - self.deltafile = 'prestodelta.xml' self.num_deltas = 1 # number of older versions to delta (max) self.max_delta_rpm_size = 100000000 self.update_md_path = None self.skip_stat = False self.database = True self.outputdir = None self.file_patterns = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] self.dir_patterns = ['.*bin\/.*', '^\/etc\/.*'] self.skip_symlinks = False self.pkglist = [] self.database_only = False self.primaryfile = 'primary.xml' self.filelistsfile = 'filelists.xml' self.otherfile = 'other.xml' self.repomdfile = 'repomd.xml' self.tempdir = '.repodata' self.finaldir = 'repodata' self.olddir = '.olddata' self.mdtimestamp = 0 self.directory = None self.directories = [] self.changelog_limit = None # needs to be an int or None self.unique_md_filenames = True self.additional_metadata = {} # dict of 'type':'filename' self.revision = str(int(time.time())) self.content_tags = [] # flat list of strings (like web 2.0 tags) self.distro_tags = []# [(cpeid(None allowed), human-readable-string)] self.repo_tags = []# strings, forwhatever they are worth self.read_pkgs_list = None # filepath/name to write out list of pkgs # read in this run of createrepo self.collapse_glibc_requires = True self.workers = 1 # number of workers to fork off to grab metadata from the pkgs self.worker_cmd = '/usr/share/createrepo/worker.py' #self.worker_cmd = './worker.py' # helpful when testing self.retain_old_md = 0 self.compress_type = 'compat' class SimpleMDCallBack(object): def errorlog(self, thing): print >> sys.stderr, thing def log(self, thing): print thing def progress(self, item, current, total): sys.stdout.write('\r' + ' ' * 80) sys.stdout.write("\r%d/%d - %s" % (current, total, item)) sys.stdout.flush() class MetaDataGenerator: def __init__(self, config_obj=None, callback=None): self.conf = config_obj if config_obj == None: self.conf = MetaDataConfig() if not callback: self.callback = SimpleMDCallBack() else: self.callback = callback self.ts = rpmUtils.transaction.initReadOnlyTransaction() self.pkgcount = 0 self.current_pkg = 0 self.files = [] self.rpmlib_reqs = {} self.read_pkgs = [] self.compat_compress = False if not self.conf.directory and not self.conf.directories: raise MDError, "No directory given on which to run." if self.conf.compress_type == 'compat': self.compat_compress = True self.conf.compress_type = None if not self.conf.compress_type: self.conf.compress_type = 'gz' if self.conf.compress_type not in utils._available_compression: raise MDError, "Compression %s not available: Please choose from: %s" \ % (self.conf.compress_type, ', '.join(utils._available_compression)) if not self.conf.directories: # just makes things easier later self.conf.directories = [self.conf.directory] if not self.conf.directory: # ensure we have both in the config object self.conf.directory = self.conf.directories[0] # the cachedir thing: if self.conf.cachedir: self.conf.use_cache = True # this does the dir setup we need done self._parse_directory() self._test_setup_dirs() def _parse_directory(self): """pick up the first directory given to us and make sure we know where things should go""" if os.path.isabs(self.conf.directory): self.conf.basedir = os.path.dirname(self.conf.directory) self.conf.relative_dir = os.path.basename(self.conf.directory) else: self.conf.basedir = os.path.realpath(self.conf.basedir) self.conf.relative_dir = self.conf.directory self.package_dir = os.path.join(self.conf.basedir, self.conf.relative_dir) if not self.conf.outputdir: self.conf.outputdir = os.path.join(self.conf.basedir, self.conf.relative_dir) def _test_setup_dirs(self): # start the sanity/stupidity checks for mydir in self.conf.directories: if os.path.isabs(mydir): testdir = mydir else: if mydir.startswith('../'): testdir = os.path.realpath(mydir) else: testdir = os.path.join(self.conf.basedir, mydir) if not os.path.exists(testdir): raise MDError, _('Directory %s must exist') % mydir if not os.path.isdir(testdir): raise MDError, _('%s must be a directory') % mydir if not os.access(self.conf.outputdir, os.W_OK): raise MDError, _('Directory %s must be writable.') % self.conf.outputdir temp_output = os.path.join(self.conf.outputdir, self.conf.tempdir) if not checkAndMakeDir(temp_output): raise MDError, _('Cannot create/verify %s') % temp_output temp_final = os.path.join(self.conf.outputdir, self.conf.finaldir) if not checkAndMakeDir(temp_final): raise MDError, _('Cannot create/verify %s') % temp_final if self.conf.database: # do flock test on temp_final, temp_output # if it fails raise MDError for direc in [temp_final, temp_output]: f = open(direc + '/locktest', 'w') try: fcntl.flock(f.fileno(), fcntl.LOCK_EX) except (OSError, IOError), e: raise MDError, _("Could not create exclusive lock in %s and sqlite database generation enabled. Is this path on nfs? Is your lockd running?") % direc else: f.close() os.unlink(direc + '/locktest') if self.conf.deltas: temp_delta = os.path.join(self.conf.outputdir, self.conf.delta_relative) if not checkAndMakeDir(temp_delta): raise MDError, _('Cannot create/verify %s') % temp_delta self.conf.deltadir = temp_delta if os.path.exists(os.path.join(self.conf.outputdir, self.conf.olddir)): raise MDError, _('Old data directory exists, please remove: %s') % self.conf.olddir # make sure we can write to where we want to write to: # and pickup the mdtimestamps while we're at it direcs = ['tempdir' , 'finaldir'] if self.conf.deltas: direcs.append('deltadir') for direc in direcs: filepath = os.path.join(self.conf.outputdir, getattr(self.conf, direc)) if os.path.exists(filepath): if not os.access(filepath, os.W_OK): raise MDError, _('error in must be able to write to metadata dir:\n -> %s') % filepath if self.conf.checkts: # checking for repodata/repomd.xml - not just the data dir rxml = filepath + '/repomd.xml' if os.path.exists(rxml): timestamp = os.path.getctime(rxml) if timestamp > self.conf.mdtimestamp: self.conf.mdtimestamp = timestamp if self.conf.groupfile: a = self.conf.groupfile if self.conf.split: a = os.path.join(self.package_dir, self.conf.groupfile) elif not os.path.isabs(a): a = os.path.join(self.package_dir, self.conf.groupfile) if not os.path.exists(a): raise MDError, _('Error: groupfile %s cannot be found.' % a) self.conf.groupfile = a if self.conf.cachedir: a = self.conf.cachedir if not os.path.isabs(a): a = os.path.join(self.conf.outputdir, a) if not checkAndMakeDir(a): raise MDError, _('Error: cannot open/write to cache dir %s' % a) self.conf.cachedir = a def _os_path_walk(self, top, func, arg): """Directory tree walk with callback function. copy of os.path.walk, fixes the link/stating problem """ try: names = os.listdir(top) except os.error: return func(arg, top, names) for name in names: name = os.path.join(top, name) if os.path.isdir(name): self._os_path_walk(name, func, arg) def getFileList(self, directory, ext): """Return all files in path matching ext, store them in filelist, recurse dirs. Returns a list object""" extlen = len(ext) def extension_visitor(filelist, dirname, names): for fn in names: fn = os.path.join(dirname, fn) if os.path.isdir(fn): continue if self.conf.skip_symlinks and os.path.islink(fn): continue elif fn[-extlen:].lower() == '%s' % (ext): filelist.append(fn[len(startdir):]) filelist = [] startdir = directory + '/' self._os_path_walk(startdir, extension_visitor, filelist) return filelist def errorlog(self, thing): """subclass this if you want something different....""" errorprint(thing) def checkTimeStamps(self): """check the timestamp of our target dir. If it is not newer than the repodata return False, else True""" if self.conf.checkts and self.conf.mdtimestamp: dn = os.path.join(self.conf.basedir, self.conf.directory) files = self.getFileList(dn, '.rpm') files = self.trimRpms(files) for f in files: fn = os.path.join(self.conf.basedir, self.conf.directory, f) if not os.path.exists(fn): self.callback.errorlog(_('cannot get to file: %s') % fn) if os.path.getctime(fn) > self.conf.mdtimestamp: return False return True return False def trimRpms(self, files): badrpms = [] for rpm_file in files: for glob in self.conf.excludes: if fnmatch.fnmatch(rpm_file, glob): if rpm_file not in badrpms: badrpms.append(rpm_file) for rpm_file in badrpms: if rpm_file in files: files.remove(rpm_file) return files def _setup_old_metadata_lookup(self): """sets up the .oldData object for handling the --update call. Speeds up generating updates for new metadata""" #FIXME - this only actually works for single dirs. It will only # function for the first dir passed to --split, not all of them # this needs to be fixed by some magic in readMetadata.py # using opts.pkgdirs as a list, I think. if self.conf.update: #build the paths opts = { 'verbose' : self.conf.verbose, 'sumtype' : self.conf.sumtype, 'pkgdir' : os.path.normpath(self.package_dir) } if self.conf.skip_stat: opts['do_stat'] = False if self.conf.update_md_path: norm_u_md_path = os.path.normpath(self.conf.update_md_path) u_md_repodata_path = norm_u_md_path + '/repodata' if not os.path.exists(u_md_repodata_path): msg = _('Warning: could not open update_md_path: %s') % u_md_repodata_path self.callback.errorlog(msg) old_repo_path = os.path.normpath(norm_u_md_path) else: old_repo_path = self.conf.outputdir #and scan the old repo self.oldData = readMetadata.MetadataIndex(old_repo_path, opts) def _setup_grabber(self): if not hasattr(self, '_grabber'): self._grabber = grabber.URLGrabber() return self._grabber grabber = property(fget = lambda self: self._setup_grabber()) def doPkgMetadata(self): """all the heavy lifting for the package metadata""" if self.conf.update: self._setup_old_metadata_lookup() # rpms we're going to be dealing with if self.conf.pkglist: packages = [] for pkg in self.conf.pkglist: if '://' in pkg: # remote packages.append(pkg) continue path = os.path.join(self.conf.basedir, self.conf.directory, pkg) if os.access(path, os.R_OK): packages.append(pkg) continue # not fatal, yet self.callback.errorlog('Cannot read file: %s' % path) else: packages = self.getFileList(self.package_dir, '.rpm') if not isinstance(packages, MetaSack): packages = self.trimRpms(packages) self.pkgcount = len(packages) try: self.openMetadataDocs() self.writeMetadataDocs(packages) self.closeMetadataDocs() except (IOError, OSError), e: raise MDError, _('Cannot access/write repodata files: %s') % e def openMetadataDocs(self): if self.conf.database_only: self.setup_sqlite_dbs() else: self.primaryfile = self._setupPrimary() self.flfile = self._setupFilelists() self.otherfile = self._setupOther() if self.conf.deltas: self.deltafile = self._setupDelta() def _setupPrimary(self): # setup the primary metadata file # FIXME - make this be conf.compress_type once y-m-p is fixed fpz = self.conf.primaryfile + '.' + 'gz' primaryfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, fpz) fo = compressOpen(primaryfilepath, 'w', 'gz') fo.write('\n') fo.write('' % self.pkgcount) return fo def _setupFilelists(self): # setup the filelist file # FIXME - make this be conf.compress_type once y-m-p is fixed fpz = self.conf.filelistsfile + '.' + 'gz' filelistpath = os.path.join(self.conf.outputdir, self.conf.tempdir, fpz) fo = compressOpen(filelistpath, 'w', 'gz') fo.write('\n') fo.write('' % self.pkgcount) return fo def _setupOther(self): # setup the other file # FIXME - make this be conf.compress_type once y-m-p is fixed fpz = self.conf.otherfile + '.' + 'gz' otherfilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, fpz) fo = compressOpen(otherfilepath, 'w', 'gz') fo.write('\n') fo.write('' % self.pkgcount) return fo def _setupDelta(self): # setup the other file fpz = self.conf.deltafile + '.' + self.conf.compress_type deltafilepath = os.path.join(self.conf.outputdir, self.conf.tempdir, fpz) fo = compressOpen(deltafilepath, 'w', self.conf.compress_type) fo.write('\n') fo.write('\n') return fo def read_in_package(self, rpmfile, pkgpath=None, reldir=None): """rpmfile == relative path to file from self.packge_dir""" baseurl = self.conf.baseurl if not pkgpath: pkgpath = self.package_dir if not rpmfile.strip(): raise MDError, "Blank filename passed in, skipping" if rpmfile.find("://") != -1: if not hasattr(self, 'tempdir'): self.tempdir = tempfile.mkdtemp() pkgname = os.path.basename(rpmfile) baseurl = os.path.dirname(rpmfile) reldir = self.tempdir dest = os.path.join(self.tempdir, pkgname) if not self.conf.quiet: self.callback.log('\nDownloading %s' % rpmfile) try: rpmfile = self.grabber.urlgrab(rpmfile, dest) except grabber.URLGrabError, e: raise MDError, "Unable to retrieve remote package %s: %s" % ( rpmfile, e) else: rpmfile = '%s/%s' % (pkgpath, rpmfile) external_data = { '_cachedir': self.conf.cachedir, '_baseurl': baseurl, '_reldir': reldir, '_packagenumber': self.current_pkg, '_collapse_libc_requires':self.conf.collapse_glibc_requires, } try: po = yumbased.CreateRepoPackage(self.ts, rpmfile, sumtype=self.conf.sumtype, external_data = external_data) except Errors.MiscError, e: raise MDError, "Unable to open package: %s" % e for r in po.requires_print: if r.startswith('rpmlib('): self.rpmlib_reqs[r] = 1 if po.checksum in (None, ""): raise MDError, "No Package ID found for package %s, not going to" \ " add it" % po return po def writeMetadataDocs(self, pkglist=[], pkgpath=None): if not pkglist: pkglist = self.conf.pkglist if not pkgpath: directory = self.conf.directory else: directory = pkgpath # for worker/forked model # iterate the pkglist - see which ones are handled by --update and let them # go on their merry way newpkgs = [] keptpkgs = [] if self.conf.update: # if we're in --update mode then only act on the new/changed pkgs for pkg in pkglist: self.current_pkg += 1 #see if we can pull the nodes from the old repo #print self.oldData.basenodes.keys() old_pkg = pkg if pkg.find("://") != -1: old_pkg = os.path.basename(pkg) old_po = self.oldData.getNodes(old_pkg) if old_po: # we have a match in the old metadata if self.conf.verbose: self.callback.log(_("Using data from old metadata for %s") % pkg) keptpkgs.append((pkg, old_po)) #FIXME - if we're in update and we have deltas enabled # check the presto data for this pkg and write its info back out # to our deltafile continue else: newpkgs.append(pkg) else: newpkgs = pkglist # setup our reldir if not pkgpath: reldir = os.path.join(self.conf.basedir, directory) else: reldir = pkgpath # filter out those pkgs which are not files - but are pkgobjects pkgfiles = [] for pkg in newpkgs: po = None if isinstance(pkg, YumAvailablePackage): po = pkg self.read_pkgs.append(po.localPkg()) # if we're dealing with remote pkgs - pitch it over to doing # them one at a time, for now. elif pkg.find('://') != -1: po = self.read_in_package(pkg, pkgpath=pkgpath, reldir=reldir) self.read_pkgs.append(pkg) if po: keptpkgs.append((pkg, po)) continue pkgfiles.append(pkg) keptpkgs.sort(reverse=True) # keptkgs is a list of (filename, po), pkgfiles is a list if filenames. # Need to write them in sorted(filename) order. We loop over pkgfiles, # inserting keptpkgs in right spots (using the upto argument). def save_keptpkgs(upto): while keptpkgs and (upto is None or keptpkgs[-1][0] < upto): filename, po = keptpkgs.pop() # reset baseurl in the old pkg po.basepath = self.conf.baseurl self.primaryfile.write(po.xml_dump_primary_metadata()) self.flfile.write(po.xml_dump_filelists_metadata()) self.otherfile.write(po.xml_dump_other_metadata( clog_limit=self.conf.changelog_limit)) if pkgfiles: # divide that list by the number of workers and fork off that many # workers to tmpdirs # waitfor the workers to finish and as each one comes in # open the files they created and write them out to our metadata # add up the total pkg counts and return that value self._worker_tmp_path = tempfile.mkdtemp() # setting this in the base object so we can clean it up later if self.conf.workers < 1: self.conf.workers = min(num_cpus_online(), len(pkgfiles)) pkgfiles.sort() worker_chunks = split_list_into_equal_chunks(pkgfiles, self.conf.workers) worker_cmd_dict = {} worker_jobs = {} base_worker_cmdline = [self.conf.worker_cmd, '--pkgoptions=_reldir=%s' % reldir, '--pkgoptions=_collapse_libc_requires=%s' % self.conf.collapse_glibc_requires, '--pkgoptions=_cachedir=%s' % self.conf.cachedir, '--pkgoptions=_baseurl=%s' % self.conf.baseurl, '--globalopts=clog_limit=%s' % self.conf.changelog_limit, '--globalopts=sumtype=%s' % self.conf.sumtype, ] if self.conf.quiet: base_worker_cmdline.append('--quiet') if self.conf.verbose: base_worker_cmdline.append('--verbose') for worker_num in range(self.conf.workers): pkl = self._worker_tmp_path + '/pkglist-%s' % worker_num f = open(pkl, 'w') f.write('\n'.join(worker_chunks[worker_num])) f.close() workercmdline = [] workercmdline.extend(base_worker_cmdline) workercmdline.append('--pkglist=%s/pkglist-%s' % (self._worker_tmp_path, worker_num)) worker_cmd_dict[worker_num] = workercmdline for (num, cmdline) in worker_cmd_dict.items(): if not self.conf.quiet: self.callback.log("Spawning worker %s with %s pkgs" % (num, len(worker_chunks[num]))) job = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE) worker_jobs[num] = job files = self.primaryfile, self.flfile, self.otherfile def log_messages(num): job = worker_jobs[num] while True: # check stdout and stderr for stream in select((job.stdout, job.stderr), (), ())[0]: line = stream.readline() if line: break else: return # EOF, EOF if stream is job.stdout: if line.startswith('*** '): if line == '*** \n': return True # get data, save to local files for out, size in zip(files, line[4:].split()): out.write(stream.read(int(size))) return self.callback.log('Worker %s: %s' % (num, line.rstrip())) else: self.callback.errorlog('Worker %s: %s' % (num, line.rstrip())) err = 0 for i, pkg in enumerate(pkgfiles): # insert cached packages save_keptpkgs(pkg) # save output to local files if log_messages(i % self.conf.workers): err += 1 for (num, job) in worker_jobs.items(): # process remaining messages on stderr log_messages(num) if job.wait() != 0: msg = "Worker exited with non-zero value: %s. Fatal." % job.returncode self.callback.errorlog(msg) raise MDError, msg if not self.conf.quiet: self.callback.log("Workers Finished") if err: raise MDError, "Failed to process %d package(s)." % err for pkgfile in pkgfiles: if self.conf.deltas: try: po = self.read_in_package(pkgfile, pkgpath=pkgpath, reldir=reldir) self._do_delta_rpm_package(po) except MDError, e: errorprint(e) continue self.read_pkgs.append(pkgfile) save_keptpkgs(None) # append anything left return self.current_pkg def closeMetadataDocs(self): # save them up to the tmp locations: if not self.conf.quiet: self.callback.log(_('Saving Primary metadata')) if self.conf.database_only: self.md_sqlite.pri_cx.close() else: self.primaryfile.write('\n') self.primaryfile.close() if not self.conf.quiet: self.callback.log(_('Saving file lists metadata')) if self.conf.database_only: self.md_sqlite.file_cx.close() else: self.flfile.write('\n') self.flfile.close() if not self.conf.quiet: self.callback.log(_('Saving other metadata')) if self.conf.database_only: self.md_sqlite.other_cx.close() else: self.otherfile.write('\n') self.otherfile.close() if self.conf.deltas: deltam_st = time.time() if not self.conf.quiet: self.callback.log(_('Saving delta metadata')) self.deltafile.write(self.generate_delta_xml()) self.deltafile.write('\n') self.deltafile.close() if self.conf.profile: self.callback.log('deltam time: %0.3f' % (time.time() - deltam_st)) def _do_delta_rpm_package(self, pkg): """makes the drpms, if possible, for this package object. returns the presto/delta xml metadata as a string """ drpm_pkg_time = time.time() # duck and cover if the pkg.size is > whatever if int(pkg.size) > self.conf.max_delta_rpm_size: if not self.conf.quiet: self.callback.log("Skipping %s package " \ "that is > max_delta_rpm_size" % pkg) return # generate a list of all the potential 'old rpms' opd = self._get_old_package_dict() # for each of our old_package_paths - # make a drpm from the newest of that pkg # get list of potential candidates which are likely to match for d in self.conf.oldpackage_paths: pot_cand = [] if d not in opd: continue for fn in opd[d]: if os.path.basename(fn).startswith(pkg.name): pot_cand.append(fn) candidates = [] for fn in pot_cand: try: thispo = yumbased.CreateRepoPackage(self.ts, fn, sumtype=self.conf.sumtype) except Errors.MiscError, e: continue if (thispo.name, thispo.arch) != (pkg.name, pkg.arch): # not the same, doesn't matter continue if thispo == pkg: #exactly the same, doesn't matter continue if thispo.EVR >= pkg.EVR: # greater or equal, doesn't matter continue candidates.append(thispo) candidates.sort() candidates.reverse() for delta_p in candidates[0:self.conf.num_deltas]: #make drpm of pkg and delta_p dt_st = time.time() drpmfn = deltarpms.create_drpm(delta_p, pkg, self.conf.deltadir) if not self.conf.quiet or self.conf.profile: self.callback.log('created drpm from %s to %s: %s in %0.3f' % ( delta_p, pkg, drpmfn, (time.time() - dt_st))) if self.conf.profile: self.callback.log('total drpm time for %s: %0.3f' % (pkg, (time.time() - drpm_pkg_time))) def _get_old_package_dict(self): if hasattr(self, '_old_package_dict'): return self._old_package_dict self._old_package_dict = {} for d in self.conf.oldpackage_paths: for f in self.getFileList(d, '.rpm'): fp = d + '/' + f fpstat = os.stat(fp) if int(fpstat[stat.ST_SIZE]) > self.conf.max_delta_rpm_size: self.callback.log("Skipping %s package " \ "that is > max_delta_rpm_size" % f) continue if not self._old_package_dict.has_key(d): self._old_package_dict[d] = [] self._old_package_dict[d].append(d + '/' + f) return self._old_package_dict def generate_delta_xml(self): """take the delta rpm output dir, process all the drpm files produce the text output for the presto/delta xml metadata""" # go through the drpm dir # for each file -store the drpm info in a dict based on its target. Just # appending the output. for each of the keys in the dict, return # the tag for the target + each of the drpm infos + closure for the target # tag targets = {} results = [] for drpm_fn in self.getFileList(self.conf.deltadir, '.drpm'): drpm_rel_fn = os.path.normpath(self.conf.delta_relative + '/' + drpm_fn) # this is annoying drpm_po = yumbased.CreateRepoPackage(self.ts, self.conf.deltadir + '/' + drpm_fn, sumtype=self.conf.sumtype) drpm = deltarpms.DeltaRPMPackage(drpm_po, self.conf.outputdir, drpm_rel_fn) if not targets.has_key(drpm_po.pkgtup): targets[drpm_po.pkgtup] = [] targets[drpm_po.pkgtup].append(drpm.xml_dump_metadata()) for (n, a, e, v, r) in targets.keys(): results.append(""" \n""" % ( n, e, v, r, a)) results.extend(targets[(n,a,e,v,r)]) # for src in targets[(n, a, e, v, r)]: # results.append(src) results.append(" \n") return ' '.join(results) def _createRepoDataObject(self, mdfile, mdtype, compress=True, compress_type=None, attribs={}): """return random metadata as RepoData object to be added to RepoMD mdfile = complete path to file mdtype = the metadata type to use compress = compress the file before including it """ # copy the file over here sfile = os.path.basename(mdfile) fo = open(mdfile, 'r') outdir = os.path.join(self.conf.outputdir, self.conf.tempdir) if not compress_type: compress_type = self.conf.compress_type if compress: sfile = '%s.%s' % (sfile, compress_type) outfn = os.path.join(outdir, sfile) output = compressOpen(outfn, mode='wb', compress_type=compress_type) else: outfn = os.path.join(outdir, sfile) output = open(outfn, 'w') output.write(fo.read()) output.close() fo.seek(0) open_csum = misc.checksum(self.conf.sumtype, fo) fo.close() if self.conf.unique_md_filenames: (csum, outfn) = checksum_and_rename(outfn, self.conf.sumtype) sfile = os.path.basename(outfn) else: if compress: csum = misc.checksum(self.conf.sumtype, outfn) else: csum = open_csum thisdata = RepoData() thisdata.type = mdtype thisdata.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, sfile)) thisdata.checksum = (self.conf.sumtype, csum) if compress: thisdata.openchecksum = (self.conf.sumtype, open_csum) thisdata.size = str(os.stat(outfn).st_size) thisdata.timestamp = str(int(os.stat(outfn).st_mtime)) for (k, v) in attribs.items(): setattr(thisdata, k, str(v)) return thisdata def doRepoMetadata(self): """wrapper to generate the repomd.xml file that stores the info on the other files""" repomd = RepoMD('repoid') repomd.revision = self.conf.revision repopath = os.path.join(self.conf.outputdir, self.conf.tempdir) repofilepath = os.path.join(repopath, self.conf.repomdfile) if self.conf.content_tags: repomd.tags['content'] = self.conf.content_tags if self.conf.distro_tags: repomd.tags['distro'] = self.conf.distro_tags # NOTE - test out the cpeid silliness here if self.conf.repo_tags: repomd.tags['repo'] = self.conf.repo_tags sumtype = self.conf.sumtype workfiles = [(self.conf.otherfile, 'other',), (self.conf.filelistsfile, 'filelists'), (self.conf.primaryfile, 'primary')] if self.conf.deltas: workfiles.append((self.conf.deltafile, 'prestodelta')) if self.conf.database: if not self.conf.quiet: self.callback.log('Generating sqlite DBs') try: dbversion = str(sqlitecachec.DBVERSION) except AttributeError: dbversion = '9' #FIXME - in theory some sort of try/except here rp = sqlitecachec.RepodataParserSqlite(repopath, repomd.repoid, None) for (rpm_file, ftype) in workfiles: # when we fix y-m-p and non-gzipped xml files - then we can make this just add # self.conf.compress_type if ftype in ('other', 'filelists', 'primary'): rpm_file = rpm_file + '.' + 'gz' elif rpm_file.find('.') != -1 and rpm_file.split('.')[-1] not in _available_compression: rpm_file = rpm_file + '.' + self.conf.compress_type complete_path = os.path.join(repopath, rpm_file) zfo = compressOpen(complete_path) # This is misc.checksum() done locally so we can get the size too. data = misc.Checksums([sumtype]) while data.read(zfo, 2**16): pass uncsum = data.hexdigest(sumtype) unsize = len(data) zfo.close() csum = misc.checksum(sumtype, complete_path) timestamp = os.stat(complete_path)[8] db_csums = {} db_compressed_sums = {} if self.conf.database: if ftype in ['primary', 'filelists', 'other']: if self.conf.verbose: self.callback.log("Starting %s db creation: %s" % (ftype, time.ctime())) if ftype == 'primary': #FIXME - in theory some sort of try/except here # TypeError appears to be raised, sometimes :( rp.getPrimary(complete_path, csum) elif ftype == 'filelists': #FIXME and here rp.getFilelists(complete_path, csum) elif ftype == 'other': #FIXME and here rp.getOtherdata(complete_path, csum) if ftype in ['primary', 'filelists', 'other']: tmp_result_name = '%s.xml.gz.sqlite' % ftype tmp_result_path = os.path.join(repopath, tmp_result_name) good_name = '%s.sqlite' % ftype resultpath = os.path.join(repopath, good_name) # compat compression for rhel5 compatibility from fedora :( compress_type = self.conf.compress_type if self.compat_compress: compress_type = 'bz2' # rename from silly name to not silly name os.rename(tmp_result_path, resultpath) compressed_name = '%s.%s' % (good_name, compress_type) result_compressed = os.path.join(repopath, compressed_name) db_csums[ftype] = misc.checksum(sumtype, resultpath) # compress the files compressFile(resultpath, result_compressed, compress_type) # csum the compressed file db_compressed_sums[ftype] = misc.checksum(sumtype, result_compressed) # timestamp+size the uncompressed file un_stat = os.stat(resultpath) # remove the uncompressed file os.unlink(resultpath) if self.conf.unique_md_filenames: csum_compressed_name = '%s-%s.%s' % ( db_compressed_sums[ftype], good_name, compress_type) csum_result_compressed = os.path.join(repopath, csum_compressed_name) os.rename(result_compressed, csum_result_compressed) result_compressed = csum_result_compressed compressed_name = csum_compressed_name # timestamp+size the compressed file db_stat = os.stat(result_compressed) # add this data as a section to the repomdxml db_data_type = '%s_db' % ftype data = RepoData() data.type = db_data_type data.location = (self.conf.baseurl, os.path.join(self.conf.finaldir, compressed_name)) data.checksum = (sumtype, db_compressed_sums[ftype]) data.timestamp = str(int(db_stat.st_mtime)) data.size = str(db_stat.st_size) data.opensize = str(un_stat.st_size) data.openchecksum = (sumtype, db_csums[ftype]) data.dbversion = dbversion if self.conf.verbose: self.callback.log("Ending %s db creation: %s" % (ftype, time.ctime())) repomd.repoData[data.type] = data data = RepoData() data.type = ftype data.checksum = (sumtype, csum) data.timestamp = str(int(timestamp)) data.size = str(os.stat(os.path.join(repopath, rpm_file)).st_size) data.opensize = str(unsize) data.openchecksum = (sumtype, uncsum) if self.conf.unique_md_filenames: if ftype in ('primary', 'filelists', 'other'): compress = 'gz' else: compress = self.conf.compress_type main_name = '.'.join(rpm_file.split('.')[:-1]) res_file = '%s-%s.%s' % (csum, main_name, compress) orig_file = os.path.join(repopath, rpm_file) dest_file = os.path.join(repopath, res_file) os.rename(orig_file, dest_file) else: res_file = rpm_file rpm_file = res_file href = os.path.join(self.conf.finaldir, rpm_file) data.location = (self.conf.baseurl, href) repomd.repoData[data.type] = data if not self.conf.quiet and self.conf.database: self.callback.log('Sqlite DBs complete') if self.conf.groupfile is not None: mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group_gz') repomd.repoData[mdcontent.type] = mdcontent mdcontent = self._createRepoDataObject(self.conf.groupfile, 'group', compress=False) repomd.repoData[mdcontent.type] = mdcontent if self.conf.additional_metadata: for md_type, md_file in self.conf.additional_metadata.items(): mdcontent = self._createRepoDataObject(md_file, md_type) repomd.repoData[mdcontent.type] = mdcontent # FIXME - disabled until we decide how best to use this #if self.rpmlib_reqs: # rpmlib = reporoot.newChild(rpmns, 'lib', None) # for r in self.rpmlib_reqs.keys(): # req = rpmlib.newChild(rpmns, 'requires', r) # save it down try: fo = open(repofilepath, 'w') fo.write(repomd.dump_xml()) fo.close() except (IOError, OSError, TypeError), e: self.callback.errorlog( _('Error saving temp file for repomd.xml: %s') % repofilepath) self.callback.errorlog('Error was: %s') % str(e) fo.close() raise MDError, 'Could not save temp file: %s' % repofilepath def doFinalMove(self): """move the just-created repodata from .repodata to repodata also make sure to preserve any files we didn't mess with in the metadata dir""" output_final_dir = os.path.join(self.conf.outputdir, self.conf.finaldir) output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir) if os.path.exists(output_final_dir): try: os.rename(output_final_dir, output_old_dir) except: raise MDError, _('Error moving final %s to old dir %s' % ( output_final_dir, output_old_dir)) output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir) try: os.rename(output_temp_dir, output_final_dir) except: # put the old stuff back os.rename(output_old_dir, output_final_dir) raise MDError, _('Error moving final metadata into place') for f in ['primaryfile', 'filelistsfile', 'otherfile', 'repomdfile', 'groupfile']: if getattr(self.conf, f): fn = os.path.basename(getattr(self.conf, f)) else: continue oldfile = os.path.join(output_old_dir, fn) if os.path.exists(oldfile): try: os.remove(oldfile) except OSError, e: raise MDError, _( 'Could not remove old metadata file: %s: %s') % (oldfile, e) old_to_remove = [] old_pr = [] old_fl = [] old_ot = [] old_pr_db = [] old_fl_db = [] old_ot_db = [] for f in os.listdir(output_old_dir): oldfile = os.path.join(output_old_dir, f) finalfile = os.path.join(output_final_dir, f) for (end,lst) in (('-primary.sqlite', old_pr_db), ('-primary.xml', old_pr), ('-filelists.sqlite', old_fl_db), ('-filelists.xml', old_fl), ('-other.sqlite', old_ot_db), ('-other.xml', old_ot)): fn = '.'.join(f.split('.')[:-1]) if fn.endswith(end): lst.append(oldfile) break # make a list of the old metadata files we don't want to remove. for lst in (old_pr, old_fl, old_ot, old_pr_db, old_fl_db, old_ot_db): sortlst = sorted(lst, key=lambda x: os.path.getmtime(x), reverse=True) for thisf in sortlst[self.conf.retain_old_md:]: old_to_remove.append(thisf) for f in os.listdir(output_old_dir): oldfile = os.path.join(output_old_dir, f) finalfile = os.path.join(output_final_dir, f) fn = '.'.join(f.split('.')[:-1]) if fn in ('filelists.sqlite', 'other.sqlite', 'primary.sqlite') or oldfile in old_to_remove: try: os.remove(oldfile) except (OSError, IOError), e: raise MDError, _( 'Could not remove old metadata file: %s: %s') % (oldfile, e) continue if os.path.exists(finalfile): # Hmph? Just leave it alone, then. try: if os.path.isdir(oldfile): shutil.rmtree(oldfile) else: os.remove(oldfile) except OSError, e: raise MDError, _( 'Could not remove old metadata file: %s: %s') % (oldfile, e) else: try: os.rename(oldfile, finalfile) except OSError, e: msg = _('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile) msg += _('Error was %s') % e raise MDError, msg self._cleanup_tmp_repodata_dir() self._cleanup_update_tmp_dir() self._write_out_read_pkgs_list() def _cleanup_update_tmp_dir(self): if not self.conf.update: return shutil.rmtree(self.oldData._repo.basecachedir, ignore_errors=True) shutil.rmtree(self.oldData._repo.base_persistdir, ignore_errors=True) def _write_out_read_pkgs_list(self): # write out the read_pkgs_list file with self.read_pkgs if self.conf.read_pkgs_list: try: fo = open(self.conf.read_pkgs_list, 'w') fo.write('\n'.join(self.read_pkgs)) fo.flush() fo.close() except (OSError, IOError), e: self.errorlog(_('Could not write out readpkgs list: %s') % self.conf.read_pkgs_list) self.errorlog(_('Error was %s') % e) def _cleanup_tmp_repodata_dir(self): output_old_dir = os.path.join(self.conf.outputdir, self.conf.olddir) output_temp_dir = os.path.join(self.conf.outputdir, self.conf.tempdir) for dirbase in (self.conf.olddir, self.conf.tempdir): dirpath = os.path.join(self.conf.outputdir, dirbase) if os.path.exists(dirpath): try: os.rmdir(dirpath) except OSError, e: self.errorlog(_('Could not remove temp metadata dir: %s') % dirbase) self.errorlog(_('Error was %s') % e) self.errorlog(_('Please clean up this directory manually.')) # our worker tmp path if hasattr(self, '_worker_tmp_path') and os.path.exists(self._worker_tmp_path): shutil.rmtree(self._worker_tmp_path, ignore_errors=True) def setup_sqlite_dbs(self, initdb=True): """sets up the sqlite dbs w/table schemas and db_infos""" destdir = os.path.join(self.conf.outputdir, self.conf.tempdir) try: self.md_sqlite = MetaDataSqlite(destdir) except sqlite.OperationalError, e: raise MDError, _('Cannot create sqlite databases: %s.\n'\ 'Maybe you need to clean up a .repodata dir?') % e class SplitMetaDataGenerator(MetaDataGenerator): """takes a series of dirs and creates repodata for all of them most commonly used with -u media:// - if no outputdir is specified it will create the repodata in the first dir in the list of dirs """ def __init__(self, config_obj=None, callback=None): MetaDataGenerator.__init__(self, config_obj=config_obj, callback=None) def _getFragmentUrl(self, url, fragment): import urlparse urlparse.uses_fragment.append('media') if not url: return url (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url) return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment))) def doPkgMetadata(self): """all the heavy lifting for the package metadata""" if len(self.conf.directories) == 1: MetaDataGenerator.doPkgMetadata(self) return if self.conf.update: self._setup_old_metadata_lookup() filematrix = {} for mydir in self.conf.directories: if os.path.isabs(mydir): thisdir = mydir else: if mydir.startswith('../'): thisdir = os.path.realpath(mydir) else: thisdir = os.path.join(self.conf.basedir, mydir) filematrix[mydir] = self.getFileList(thisdir, '.rpm') # pkglist is a bit different for split media, as we have to know # which dir. it belongs to. So we walk the dir. and then filter. # We could be faster by not walking the dir. ... but meh. if self.conf.pkglist: pkglist = set(self.conf.pkglist) pkgs = [] for fname in filematrix[mydir]: if fname not in pkglist: continue pkgs.append(fname) filematrix[mydir] = pkgs self.trimRpms(filematrix[mydir]) self.pkgcount += len(filematrix[mydir]) mediano = 1 self.current_pkg = 0 self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano) try: self.openMetadataDocs() for mydir in self.conf.directories: self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, mediano) self.writeMetadataDocs(filematrix[mydir], mydir) mediano += 1 self.conf.baseurl = self._getFragmentUrl(self.conf.baseurl, 1) self.closeMetadataDocs() except (IOError, OSError), e: raise MDError, _('Cannot access/write repodata files: %s') % e class MetaDataSqlite(object): def __init__(self, destdir): self.pri_sqlite_file = os.path.join(destdir, 'primary.sqlite') self.pri_cx = sqlite.Connection(self.pri_sqlite_file) self.file_sqlite_file = os.path.join(destdir, 'filelists.sqlite') self.file_cx = sqlite.Connection(self.file_sqlite_file) self.other_sqlite_file = os.path.join(destdir, 'other.sqlite') self.other_cx = sqlite.Connection(self.other_sqlite_file) self.primary_cursor = self.pri_cx.cursor() self.filelists_cursor = self.file_cx.cursor() self.other_cursor = self.other_cx.cursor() self.create_primary_db() self.create_filelists_db() self.create_other_db() def create_primary_db(self): # make the tables schema = [ """PRAGMA synchronous="OFF";""", """pragma locking_mode="EXCLUSIVE";""", """CREATE TABLE conflicts ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""", """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", """CREATE TABLE files ( name TEXT, type TEXT, pkgKey INTEGER);""", """CREATE TABLE obsoletes ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""", """CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT, name TEXT, arch TEXT, version TEXT, epoch TEXT, release TEXT, summary TEXT, description TEXT, url TEXT, time_file INTEGER, time_build INTEGER, rpm_license TEXT, rpm_vendor TEXT, rpm_group TEXT, rpm_buildhost TEXT, rpm_sourcerpm TEXT, rpm_header_start INTEGER, rpm_header_end INTEGER, rpm_packager TEXT, size_package INTEGER, size_installed INTEGER, size_archive INTEGER, location_href TEXT, location_base TEXT, checksum_type TEXT);""", """CREATE TABLE provides ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER );""", """CREATE TABLE requires ( name TEXT, flags TEXT, epoch TEXT, version TEXT, release TEXT, pkgKey INTEGER , pre BOOL DEFAULT FALSE);""", """CREATE INDEX filenames ON files (name);""", """CREATE INDEX packageId ON packages (pkgId);""", """CREATE INDEX packagename ON packages (name);""", """CREATE INDEX pkgconflicts on conflicts (pkgKey);""", """CREATE INDEX pkgobsoletes on obsoletes (pkgKey);""", """CREATE INDEX pkgprovides on provides (pkgKey);""", """CREATE INDEX pkgrequires on requires (pkgKey);""", """CREATE INDEX providesname ON provides (name);""", """CREATE INDEX requiresname ON requires (name);""", """CREATE TRIGGER removals AFTER DELETE ON packages BEGIN DELETE FROM files WHERE pkgKey = old.pkgKey; DELETE FROM requires WHERE pkgKey = old.pkgKey; DELETE FROM provides WHERE pkgKey = old.pkgKey; DELETE FROM conflicts WHERE pkgKey = old.pkgKey; DELETE FROM obsoletes WHERE pkgKey = old.pkgKey; END;""", """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION, ] for cmd in schema: executeSQL(self.primary_cursor, cmd) def create_filelists_db(self): schema = [ """PRAGMA synchronous="OFF";""", """pragma locking_mode="EXCLUSIVE";""", """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", """CREATE TABLE filelist ( pkgKey INTEGER, dirname TEXT, filenames TEXT, filetypes TEXT);""", """CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);""", """CREATE INDEX dirnames ON filelist (dirname);""", """CREATE INDEX keyfile ON filelist (pkgKey);""", """CREATE INDEX pkgId ON packages (pkgId);""", """CREATE TRIGGER remove_filelist AFTER DELETE ON packages BEGIN DELETE FROM filelist WHERE pkgKey = old.pkgKey; END;""", """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION, ] for cmd in schema: executeSQL(self.filelists_cursor, cmd) def create_other_db(self): schema = [ """PRAGMA synchronous="OFF";""", """pragma locking_mode="EXCLUSIVE";""", """CREATE TABLE changelog ( pkgKey INTEGER, author TEXT, date INTEGER, changelog TEXT);""", """CREATE TABLE db_info (dbversion INTEGER, checksum TEXT);""", """CREATE TABLE packages ( pkgKey INTEGER PRIMARY KEY, pkgId TEXT);""", """CREATE INDEX keychange ON changelog (pkgKey);""", """CREATE INDEX pkgId ON packages (pkgId);""", """CREATE TRIGGER remove_changelogs AFTER DELETE ON packages BEGIN DELETE FROM changelog WHERE pkgKey = old.pkgKey; END;""", """INSERT into db_info values (%s, 'direct_create');""" % sqlitecachec.DBVERSION, ] for cmd in schema: executeSQL(self.other_cursor, cmd) createrepo-0.10.3/createrepo/Makefile0000664000175000017500000000304612271721201017437 0ustar zpavlaszpavlasPYTHON=python PACKAGE = $(shell basename `pwd`) PYFILES = $(wildcard *.py) PYVER := $(shell $(PYTHON) -c 'import sys; print "%.3s" %(sys.version)') PYSYSDIR := $(shell $(PYTHON) -c 'import sys; print sys.prefix') PYLIBDIR = $(PYSYSDIR)/lib/python$(PYVER) PKGDIR = $(PYLIBDIR)/site-packages/$(PKGNAME) SHELL = /bin/sh top_srcdir = .. srcdir = ../$(PKGNAME) prefix = /usr exec_prefix = ${prefix} bindir = ${exec_prefix}/bin sbindir = ${exec_prefix}/sbin libexecdir = ${exec_prefix}/libexec datadir = ${prefix}/share sysconfdir = ${prefix}/etc sharedstatedir = ${prefix}/com localstatedir = ${prefix}/var libdir = ${exec_prefix}/lib infodir = ${prefix}/info docdir = includedir = ${prefix}/include oldincludedir = /usr/include mandir = ${datadir}/man pkgdatadir = $(datadir)/$(PKGNAME) pkglibdir = $(libdir)/$(PKGNAME) pkgincludedir = $(includedir)/$(PKGNAME) top_builddir = ../ all: echo "Nothing to do" clean: rm -f *.pyc *.pyo *~ install: mkdir -p $(DESTDIR)/$(PKGDIR) for p in $(PYFILES) ; do \ install -m 644 $$p $(DESTDIR)/$(PKGDIR)/$$p; \ done $(PYTHON) -c "import compileall; compileall.compile_dir('$(DESTDIR)/$(PKGDIR)', 1, '$(PKGDIR)', 1)" distfiles: distdir=$(PKGNAME)-$(VERSION); \ mkdir $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME);\ cp \ $(srcdir)/$(PYFILES) \ $(srcdir)/Makefile \ $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME) dailyfiles: distdir=$(PKGNAME); \ mkdir $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME);\ cp \ $(srcdir)/$(PYFILES) \ $(srcdir)/__init__.py \ $(srcdir)/Makefile \ $(top_srcdir)/.disttmp/$$distdir/$(PKGNAME) createrepo-0.10.3/createrepo/deltarpms.py0000664000175000017500000001062712271721201020347 0ustar zpavlaszpavlas#!/usr/bin/python -tt # util functions for deltarpms # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # copyright 2009 - Red Hat import os.path import commands from yum import misc import deltarpm from utils import MDError class DeltaRPMPackage: """each drpm is one object, you pass it a drpm file it opens the file, and pulls the information out in bite-sized chunks :) """ mode_cache = {} def __init__(self, po, basedir, filename): try: stats = os.stat(os.path.join(basedir, filename)) self.size = stats[6] self.mtime = stats[8] del stats except OSError, e: raise MDError, "Error Stat'ing file %s%s" % (basedir, filename) self.csum_type = 'sha256' self.relativepath = filename self.po = po fd = os.open(self.po.localpath, os.O_RDONLY) os.lseek(fd, 0, 0) fo = os.fdopen(fd, 'rb') self.csum = misc.checksum(self.csum_type, fo) del fo del fd self._getDRPMInfo(os.path.join(basedir, filename)) def _stringToNEVR(self, string): i = string.rfind("-", 0, string.rfind("-")-1) name = string[:i] (epoch, ver, rel) = self._stringToVersion(string[i+1:]) return (name, epoch, ver, rel) def _getLength(self, in_data): length = 0 for val in in_data: length = length * 256 length += ord(val) return length def _getDRPMInfo(self, filename): d = deltarpm.readDeltaRPM(filename) self.oldnevrstring = d['old_nevr'] self.oldnevr = self._stringToNEVR(d['old_nevr']) self.sequence = d['seq'] def _stringToVersion(self, strng): i = strng.find(':') if i != -1: epoch = strng[:i] else: epoch = '0' j = strng.find('-') if j != -1: if strng[i + 1:j] == '': version = None else: version = strng[i + 1:j] release = strng[j + 1:] else: if strng[i + 1:] == '': version = None else: version = strng[i + 1:] release = None return (epoch, version, release) def xml_dump_metadata(self): """takes an xml doc object and a package metadata entry node, populates a package node with the md information""" (oldname, oldepoch, oldver, oldrel) = self.oldnevr sequence = "%s-%s" % (self.oldnevrstring, self.sequence) delta_tag = """ %s %s %s %s \n""" % (oldepoch, oldver, oldrel, self.relativepath, sequence, self.size, self.csum_type, self.csum) return delta_tag def create_drpm(old_pkg, new_pkg, destdir): """make a drpm file, if possible. returns None if nothing could be created""" drpmfn = '%s-%s-%s_%s-%s.%s.drpm' % (old_pkg.name, old_pkg.ver, old_pkg.release, new_pkg.ver, new_pkg.release, old_pkg.arch) delta_rpm_path = os.path.join(destdir, drpmfn) delta_command = '/usr/bin/makedeltarpm %s %s %s' % (old_pkg.localpath, new_pkg.localpath, delta_rpm_path) if not os.path.exists(delta_rpm_path): #TODO - check/verify the existing one a bit? (code, out) = commands.getstatusoutput(delta_command) if code: print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (old_pkg.name, code, out) return None return delta_rpm_path createrepo-0.10.3/Makefile0000664000175000017500000001141112271721200015300 0ustar zpavlaszpavlasPKGNAME = createrepo ALIASES = mergerepo modifyrepo genpkgmetadata.py mergerepo.py modifyrepo.py VERSION=$(shell awk '/Version:/ { print $$2 }' ${PKGNAME}.spec) RELEASE=$(shell awk '/Release:/ { print $$2 }' ${PKGNAME}.spec) CVSTAG=createrepo-$(subst .,_,$(VERSION)-$(RELEASE)) PYTHON=python SUBDIRS = $(PKGNAME) bin docs PYFILES = $(wildcard *.py) SHELL = /bin/sh top_srcdir = . srcdir = . prefix = /usr exec_prefix = ${prefix} bindir = ${exec_prefix}/bin sbindir = ${exec_prefix}/sbin libexecdir = ${exec_prefix}/libexec datadir = ${prefix}/share sysconfdir = ${prefix}/etc sharedstatedir = ${prefix}/com localstatedir = ${prefix}/var libdir = ${exec_prefix}/lib infodir = ${prefix}/info docdir = includedir = ${prefix}/include oldincludedir = /usr/include mandir = ${prefix}/share/man compdir = $(shell pkg-config --variable=completionsdir bash-completion) compdir := $(or $(compdir), "/etc/bash_completion.d") pkgdatadir = $(datadir)/$(PKGNAME) pkglibdir = $(libdir)/$(PKGNAME) pkgincludedir = $(includedir)/$(PKGNAME) top_builddir = # all dirs DIRS = $(DESTDIR)$(bindir) $(DESTDIR)$(compdir) \ $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(mandir) # INSTALL scripts INSTALL = install -p --verbose INSTALL_BIN = $(INSTALL) -m 755 INSTALL_DIR = $(INSTALL) -m 755 -d INSTALL_DATA = $(INSTALL) -m 644 INSTALL_MODULES = $(INSTALL) -m 755 -D RM = rm -f MODULES = $(srcdir)/genpkgmetadata.py \ $(srcdir)/modifyrepo.py \ $(srcdir)/mergerepo.py \ $(srcdir)/worker.py .SUFFIXES: .py .pyc .py.pyc: python -c "import py_compile; py_compile.compile($*.py)" all: $(MODULES) for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir VERSION=$(VERSION) PKGNAME=$(PKGNAME) DESTDIR=$(DESTDIR); \ done check: pychecker $(MODULES) || exit 0 install: all installdirs $(INSTALL_MODULES) $(srcdir)/$(MODULES) $(DESTDIR)$(pkgdatadir) $(INSTALL_DATA) $(PKGNAME).bash $(DESTDIR)$(compdir)/$(PKGNAME) (cd $(DESTDIR)$(compdir); for n in $(ALIASES); do ln -s $(PKGNAME) $$n; done) for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir install VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ done installdirs: for dir in $(DIRS) ; do \ $(INSTALL_DIR) $$dir ; \ done uninstall: for module in $(MODULES) ; do \ $(RM) $(pkgdatadir)/$$module ; \ done for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir uninstall VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ done clean: $(RM) *.pyc *.pyo for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir clean VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ done distclean: clean $(RM) -r .libs $(RM) core $(RM) *~ for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir distclean VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ done pylint: @pylint --rcfile=test/createrepo-pylintrc *.py createrepo pylint-short: @pylint -r n --rcfile=test/createrepo-pylintrc *.py createrepo mostlyclean: $(MAKE) clean maintainer-clean: $(MAKE) distclean $(RM) $(srcdir)/configure changelog: git log --pretty --numstat --summary | git2cl > ChangeLog dist: olddir=`pwd`; \ distdir=$(PKGNAME)-$(VERSION); \ $(RM) -r .disttmp; \ $(INSTALL_DIR) .disttmp; \ $(INSTALL_DIR) .disttmp/$$distdir; \ $(MAKE) distfiles distdir=$(PKGNAME)-$(VERSION); \ cd .disttmp; \ tar -cvz > ../$$distdir.tar.gz $$distdir; \ cd $$olddir $(RM) -r .disttmp daily: olddir=`pwd`; \ distdir=$(PKGNAME); \ $(RM) -r .disttmp; \ $(INSTALL_DIR) .disttmp; \ $(INSTALL_DIR) .disttmp/$$distdir; \ $(MAKE) dailyfiles day=`/bin/date +%Y%m%d`; \ distdir=$(PKGNAME); \ tarname=$$distdir-$$day ;\ cd .disttmp; \ perl -pi -e "s/\#DATE\#/$$day/g" $$distdir/$(PKGNAME)-daily.spec; \ echo $$day; \ tar -cvz > ../$$tarname.tar.gz $$distdir; \ cd $$olddir $(RM) -rf .disttmp dailyfiles: distdir=$(PKGNAME); \ cp \ $(srcdir)/*.py \ $(srcdir)/Makefile \ $(srcdir)/ChangeLog \ $(srcdir)/COPYING \ $(srcdir)/COPYING.lib \ $(srcdir)/README \ $(srcdir)/$(PKGNAME).spec \ $(srcdir)/$(PKGNAME).bash \ $(top_srcdir)/.disttmp/$$distdir for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir dailyfiles VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ done distfiles: distdir=$(PKGNAME)-$(VERSION); \ cp \ $(srcdir)/*.py \ $(srcdir)/Makefile \ $(srcdir)/ChangeLog \ $(srcdir)/COPYING \ $(srcdir)/COPYING.lib \ $(srcdir)/README \ $(srcdir)/$(PKGNAME).spec \ $(srcdir)/$(PKGNAME).bash \ $(top_srcdir)/.disttmp/$$distdir for subdir in $(SUBDIRS) ; do \ $(MAKE) -C $$subdir distfiles VERSION=$(VERSION) PKGNAME=$(PKGNAME); \ done archive: dist .PHONY: todo todo: @echo ---------------=========================================== @grep -n TODO\\\|FIXME `find . -type f` | grep -v grep @echo ---------------=========================================== .PHONY: all install install-strip uninstall clean distclean mostlyclean maintainer-clean info dvi dist distfiles check installcheck installdirs daily dailyfiles createrepo-0.10.3/genpkgmetadata.py0000775000175000017500000003026712271721200017203 0ustar zpavlaszpavlas#!/usr/bin/python -t # primary functions and glue for generating the repository metadata # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2004 Duke University # Portions Copyright 2009 Red Hat, Inc - # written by seth vidal skvidal at fedoraproject.org import os import sys import re from optparse import OptionParser,SUPPRESS_HELP import time import errno import createrepo from createrepo import MDError from createrepo.utils import errorprint, _ import yum.misc def parse_args(args, conf): """ Parse the command line args. return a config object. Sanity check all the things being passed in. """ def_workers = os.nice(0) if def_workers > 0: def_workers = 1 # We are niced, so just use a single worker. else: def_workers = 0 # zoooom.... _def = yum.misc._default_checksums[0] _avail = yum.misc._available_checksums parser = OptionParser(version = "createrepo %s" % createrepo.__version__) # query options parser.add_option("-q", "--quiet", default=False, action="store_true", help="output nothing except for serious errors") parser.add_option("-v", "--verbose", default=False, action="store_true", help="output more debugging info.") parser.add_option("--profile", default=False, action="store_true", help="output timing/profile info.") parser.add_option("-x", "--excludes", default=[], action="append", help="files to exclude") parser.add_option("--basedir", default=os.getcwd(), help="basedir for path to directories") parser.add_option("-u", "--baseurl", default=None, help="baseurl to append on all files") parser.add_option("-g", "--groupfile", default=None, help="path to groupfile to include in metadata") parser.add_option("-s", "--checksum", default=_def, dest='sumtype', help="specify the checksum type to use (default: %s)" % _def) parser.add_option("-p", "--pretty", default=False, action="store_true", help="make sure all xml generated is formatted") parser.add_option("-c", "--cachedir", default=None, help="set path to cache dir") parser.add_option("-C", "--checkts", default=False, action="store_true", help="check timestamps on files vs the metadata to see " \ "if we need to update") parser.add_option("-d", "--database", default=True, action="store_true", help="create sqlite database files: now default, see --no-database to disable") parser.add_option("--no-database", default=False, dest="nodatabase", action="store_true", help="do not create sqlite dbs of metadata") # temporarily disabled #parser.add_option("--database-only", default=False, action="store_true", # dest='database_only', # help="Only make the sqlite databases - does not work with --update, yet") parser.add_option("--update", default=False, action="store_true", help="use the existing repodata to speed up creation of new") parser.add_option("--update-md-path", default=None, dest='update_md_path', help="use the existing repodata for --update from this path") parser.add_option("--skip-stat", dest='skip_stat', default=False, help="skip the stat() call on a --update, assumes if the file" \ "name is the same then the file is still the same " \ "(only use this if you're fairly trusting or gullible)", action="store_true") parser.add_option("--split", default=False, action="store_true", help="generate split media") parser.add_option("-i", "--pkglist", default=None, help="use only the files listed in this file from the " \ "directory specified") parser.add_option("-n", "--includepkg", default=[], action="append", help="add this pkg to the list - can be specified multiple times") parser.add_option("-o", "--outputdir", default=None, help=" = optional directory to output to") parser.add_option("-S", "--skip-symlinks", dest="skip_symlinks", default=False, action="store_true", help="ignore symlinks of packages") parser.add_option("--changelog-limit", dest="changelog_limit", default=None, help="only import the last N changelog entries") parser.add_option("--unique-md-filenames", dest="unique_md_filenames", help="include the file's checksum in the filename, helps with proxies (default)", default=True, action="store_true") parser.add_option("--simple-md-filenames", dest="unique_md_filenames", help="do not include the file's checksum in the filename", action="store_false") parser.add_option("--retain-old-md", default=0, type='int', dest='retain_old_md', help="keep around the latest (by timestamp) N copies of the old repodata") parser.add_option("--distro", default=[], action="append", help="distro tag and optional cpeid: --distro" "'cpeid,textname'") parser.add_option("--content", default=[], dest='content_tags', action="append", help="tags for the content in the repository") parser.add_option("--repo", default=[], dest='repo_tags', action="append", help="tags to describe the repository itself") parser.add_option("--revision", default=None, help="user-specified revision for this repository") parser.add_option("--deltas", default=False, action="store_true", help="create delta rpms and metadata") parser.add_option("--oldpackagedirs", default=[], dest="oldpackage_paths", action="append", help="paths to look for older pkgs to delta against") parser.add_option("--num-deltas", default=1, dest='num_deltas', type='int', help="the number of older versions to make deltas against") parser.add_option("--read-pkgs-list", default=None, dest='read_pkgs_list', help="output the paths to the pkgs actually read useful with --update") parser.add_option("--max-delta-rpm-size", default=100000000, dest='max_delta_rpm_size', type='int', help="max size of an rpm that to run deltarpm against (in bytes)") parser.add_option("--workers", default=def_workers, dest='workers', type='int', help="number of workers to spawn to read rpms") parser.add_option("--xz", default=False, action="store_true", help=SUPPRESS_HELP) parser.add_option("--compress-type", default='compat', dest="compress_type", help="which compression type to use") (opts, argsleft) = parser.parse_args(args) if len(argsleft) > 1 and not opts.split: errorprint(_('Error: Only one directory allowed per run.')) parser.print_usage() sys.exit(1) elif len(argsleft) == 0: errorprint(_('Error: Must specify a directory to index.')) parser.print_usage() sys.exit(1) else: directories = argsleft if opts.workers >= 128: errorprint(_('Warning: More than 128 workers is a lot. Limiting.')) opts.workers = 128 if opts.sumtype == 'sha1': errorprint(_('Warning: It is more compatible to use sha instead of sha1')) if opts.sumtype != 'sha' and opts.sumtype not in _avail: errorprint(_('Error: Checksum %s not available (sha, %s)') % (opts.sumtype, ", ".join(sorted(_avail)))) sys.exit(1) if opts.split and opts.checkts: errorprint(_('--split and --checkts options are mutually exclusive')) sys.exit(1) if opts.nodatabase: opts.database = False # xz is just a shorthand for compress_type if opts.xz and opts.compress_type == 'compat': opts.compress_type='xz' # let's switch over to using the conf object - put all the opts into it for opt in parser.option_list: if opt.dest is None: # this is fairly silly continue # if it's not set, take the default from the base class if getattr(opts, opt.dest) is None: continue setattr(conf, opt.dest, getattr(opts, opt.dest)) directory = directories[0] conf.directory = directory conf.directories = directories # distro tag parsing for spec in opts.distro: if spec.find(',') == -1: conf.distro_tags.append((None, spec)) else: splitspec = spec.split(',') conf.distro_tags.append((splitspec[0], splitspec[1])) lst = [] if conf.pkglist: try: for line in open(conf.pkglist): line = line.strip() if re.match('^\s*\#.*', line) or re.match('^\s*$', line): continue lst.append(line) except EnvironmentError, e: print >> sys.stderr, e sys.exit(1) conf.pkglist = lst if conf.includepkg: conf.pkglist.extend(conf.includepkg) if conf.changelog_limit: # make sure it is an int, not a string conf.changelog_limit = int(conf.changelog_limit) return conf class MDCallBack(object): """cli callback object for createrepo""" def __init__(self): self.__show_progress = os.isatty(1) def errorlog(self, thing): """error log output""" print >> sys.stderr, thing def log(self, thing): """log output""" print thing def progress(self, item, current, total): """progress bar""" if not self.__show_progress: return beg = "%*d/%d - " % (len(str(total)), current, total) left = 80 - len(beg) sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) sys.stdout.flush() def main(args): """createrepo from cli main flow""" try: os.getcwd() except OSError, e: if e.errno != errno.ENOENT: raise print ('No getcwd() access in current directory.') sys.exit(1) start_st = time.time() conf = createrepo.MetaDataConfig() conf = parse_args(args, conf) if conf.profile: print ('start time: %0.3f' % (time.time() - start_st)) mid_st = time.time() try: if conf.split: mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack()) else: mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack()) if mdgen.checkTimeStamps(): if mdgen.conf.verbose: print _('repo is up to date') mdgen._cleanup_tmp_repodata_dir() sys.exit(0) if conf.profile: print ('mid time: %0.3f' % (time.time() - mid_st)) pm_st = time.time() mdgen.doPkgMetadata() if conf.profile: print ('pm time: %0.3f' % (time.time() - pm_st)) rm_st = time.time() mdgen.doRepoMetadata() if conf.profile: print ('rm time: %0.3f' % (time.time() - rm_st)) fm_st = time.time() mdgen.doFinalMove() if conf.profile: print ('fm time: %0.3f' % (time.time() - fm_st)) except MDError, errormsg: errorprint(_('%s') % errormsg) # cleanup tmp = os.path.join(conf.outputdir, conf.tempdir) if os.path.exists(tmp): for name in os.listdir(tmp): os.unlink(os.path.join(tmp, name)) os.rmdir(tmp) sys.exit(1) if __name__ == "__main__": if len(sys.argv) > 1: if sys.argv[1] == 'profile': import hotshot p = hotshot.Profile(os.path.expanduser("~/createrepo.prof")) p.run('main(sys.argv[2:])') p.close() else: main(sys.argv[1:]) else: main(sys.argv[1:]) createrepo-0.10.3/COPYING0000664000175000017500000004307012271721200014701 0ustar zpavlaszpavlas GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 675 Mass Ave, Cambridge, MA 02139, USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) 19yy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) 19yy name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. createrepo-0.10.3/mergerepo.py0000775000175000017500000000654012271721200016211 0ustar zpavlaszpavlas#!/usr/bin/python -tt # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2008 Red Hat, Inc - written by skvidal at fedoraproject.org # merge repos from arbitrary repo urls import sys import createrepo.merge from createrepo.utils import MDError from optparse import OptionParser #TODO: # excludes? # handle content/distro tags # support revision? def parse_args(args): """Parse our opts/args""" usage = """ mergerepo: take 2 or more repositories and merge their metadata into a new repo mergerepo --repo=url --repo=url --outputdir=/some/path""" parser = OptionParser(version = "mergerepo 0.1", usage=usage) # query options parser.add_option("-r", "--repo", dest='repos', default=[], action="append", help="repo url") parser.add_option("-a", "--archlist", default=[], action="append", help="Defaults to all arches - otherwise specify arches") parser.add_option("-d", "--database", default=True, action="store_true") parser.add_option( "--no-database", default=False, action="store_true", dest="nodatabase") parser.add_option("-o", "--outputdir", default=None, help="Location to create the repository") parser.add_option("", "--nogroups", default=False, action="store_true", help="Do not merge group(comps) metadata") parser.add_option("", "--noupdateinfo", default=False, action="store_true", help="Do not merge updateinfo metadata") parser.add_option("--compress-type", default=None, dest="compress_type", help="which compression type to use") (opts, argsleft) = parser.parse_args(args) if len(opts.repos) < 2: parser.print_usage() sys.exit(1) # sort out the comma-separated crap we somehow inherited. archlist = [] for archs in opts.archlist: for arch in archs.split(','): archlist.append(arch) opts.archlist = archlist return opts def main(args): """main""" opts = parse_args(args) rmbase = createrepo.merge.RepoMergeBase(opts.repos) if opts.archlist: rmbase.archlist = opts.archlist if opts.outputdir: rmbase.outputdir = opts.outputdir if opts.nodatabase: rmbase.mdconf.database = False if opts.nogroups: rmbase.groups = False if opts.noupdateinfo: rmbase.updateinfo = False if opts.compress_type: rmbase.mdconf.compress_type = opts.compress_type try: rmbase.merge_repos() rmbase.write_metadata() except MDError, e: print >> sys.stderr, "Could not merge repos: %s" % e sys.exit(1) if __name__ == "__main__": main(sys.argv[1:]) createrepo-0.10.3/README0000664000175000017500000000041112271721200014516 0ustar zpavlaszpavlasThis program generates a repodata dir and xml files for a repository of rpm packages. This repository is compatible with apt/yum/smart/yast and many other package-repository-related tools. run createrepo -h for usage syntax http://createrepo.baseurl.org/ createrepo-0.10.3/COPYING.lib0000664000175000017500000006350012271721200015446 0ustar zpavlaszpavlas GNU LESSER GENERAL PUBLIC LICENSE Version 2.1, February 1999 Copyright (C) 1991, 1999 Free Software Foundation, Inc. 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. [This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do these things. To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling it. And you must show them these terms so they know their rights. We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal permission to copy, distribute and/or modify the library. To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use this license for certain libraries in order to permit linking those libraries into non-free programs. When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with the library. We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain special circumstances. For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free software only, so we use the Lesser General Public License. In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU operating system, as well as its variant, the GNU/Linux operating system. Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). Each licensee is addressed as "you". A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs (which use some of those functions and data) to form executables. The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is included without limitation in the term "modification".) "Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the library. Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the Library. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) The modified work must itself be a software library. b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, in the event an application does not supply such function or table, the facility still operates, and performs whatever part of its purpose remains meaningful. (For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must be optional: if the application does not supply it, the square root function must still compute square roots.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Library. In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. This option is useful when you wish to copy part of the code of the Library into a program that is not a library. 4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place satisfies the requirement to distribute the source code, even though third parties are not compelled to copy the source along with the object code. 5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and therefore falls outside the scope of this License. However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. Section 6 states terms for distribution of such executables. When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit modification of the work for the customer's own use and reverse engineering for debugging such modifications. You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one of these things: a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application to use the modified definitions.) b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) will operate properly with a modified version of the library, if the user installs one, as long as the modified version is interface-compatible with the version that the work was made with. c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above specified materials from the same place. e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities. This must be distributed under the terms of the Sections above. b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Library or works based on it. 10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Libraries If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the ordinary General Public License). To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the library, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the library `Frob' (a library for tweaking knobs) written by James Random Hacker. , 1 April 1990 Ty Coon, President of Vice That's all there is to it! createrepo-0.10.3/ChangeLog0000664000175000017500000015763612271721200015436 0ustar zpavlaszpavlas2014-01-28 Zdenek Pavlas * createrepo --update: ignore cached rpm when checksum_type has changed. 2014-01-16 Zdenek Pavlas * catch IOErrors when loading pkglist. BZ 1044997 * Add missing int() on mtime * Fix file/dir completions for names containing spaces or tabs 2013-11-28 Zdenek Pavlas * Removing the workers=1 default breaks API users, put it back. * Make sure the "packages" attribute is always correct. * Clean up tempdir on failures. * Close lock file before unlink() * mark as 0.10.1 2013-10-18 Zdenek Pavlas * mark as 0.10 2011-01-26 Seth Vidal * createrepo.spec, createrepo/__init__.py: mark as 0.9.9 2011-01-26 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Override timestamp check on repos. for mergerepo (like repodiff). Add createrepo --workers (non)completion. Add modifyrepo option completion. 2011-01-21 James Antill * createrepo/merge.py: Override timestamp check on repos. for mergerepo (like repodiff). 2011-01-03 Seth Vidal * createrepo/__init__.py: make sure when we want to look for rpms we say .rpm not rpm b/c with the latter we catch .drpm files, too. :( 2010-11-02 Ville Skyttä * createrepo.bash: Add createrepo --workers (non)completion. 2010-11-02 Ville Skyttä * createrepo.bash: Add modifyrepo option completion. 2010-10-08 Seth Vidal * createrepo.spec, createrepo/__init__.py: - add yum 3.2.29 requirement b/c of the small change I needed to repoMDObject.py - set it to use /usr/share/createrepo/worker.py 2010-10-07 Seth Vidal * createrepo/__init__.py: remove libxml2 import from __init__.py :) 2010-10-07 Seth Vidal * createrepo/__init__.py: make createrepo use the repomd/repodata mechanism from yum for making a repomd.xml which simplifies the code dramatically since we don't have to mess with xml in here. 2010-10-07 Seth Vidal * modifyrepo.py: fix up the usage output for modifyrepo 2010-09-10 Seth Vidal * createrepo/__init__.py, worker.py: - make sure we handle remote_url pkgs correctly until we get the worker hooked up to handle them - if there are no pkgs to handle, don't launch workers with nothing to do. - give better output from the workers and have them obey -v/-q - everyone loves callbacks! 2010-09-09 Seth Vidal * Makefile, createrepo/__init__.py, createrepo/utils.py, createrepo/yumbased.py, genpkgmetadata.py, worker.py: create a worker script for createrepo so createrepo can fork off N processes to handle the md gathering from pkgs. This should speed up results on systems which have been cpubound on the createrepo process. If you're io bound it won't help you at all, and MAY make it worse. many misc issues to iron out here - not the least of which is the callback output and gathering stdout/stderr from the workers 2010-08-20 Seth Vidal * createrepo/__init__.py: handle broken locking on nfs target dirs better if database is true. - sqlite dbs don't like being made on locations without locking available. - if we know we're going to be creating dbs then we should attempt to lock before doing anything else and bail out nicely if we can't get an exclusive lock 2010-08-19 Seth Vidal * createrepo.spec: require yum 3.2.28 due to the imports in modifyrepo 2010-08-19 Seth Vidal * docs/modifyrepo.1: document --mdtype option 2010-08-19 Seth Vidal * modifyrepo.py: - add option parsing for --mdtype - use the yum repomd objects to read/write the repomd.xml - add mdtype option to RepoMetadata.add() method 2010-06-11 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Don't use /usr/bin/env ... it's evil --database is now the default for mergerepo too, have --no-database in completions instead. 2010-06-11 Seth Vidal * createrepo/__init__.py: - add option to createrepo config to collapse libc.so.6 requires - this will only work with yum 3.2.28 and beyond 2010-06-03 James Antill * modifyrepo.py: Don't use /usr/bin/env ... it's evil 2010-06-02 Ville Skyttä * createrepo.bash: --database is now the default for mergerepo too, have --no-database in completions instead. 2010-06-01 Seth Vidal * mergerepo.py: whoops - no-database shouldn't default to true! 2010-06-01 Seth Vidal * mergerepo.py: add --no-database to mergrepo, too 2010-05-31 Ville Skyttä * createrepo.bash: --database is now the default, have --no-database in completions instead. 2010-05-28 Seth Vidal * docs/createrepo.8: update the docs for --no-database 2010-05-28 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: make -d/--database the default add --no-database in case someone somewhere needs to do that 2010-04-26 Seth Vidal * createrepo/__init__.py: fixme comments about try/excepting the database creation calls due to a weird issue with locks not working on a nfs mount and createreepo tracing back with a TypeError of all things 2010-04-21 Seth Vidal * createrepo/__init__.py, createrepo/readMetadata.py: if we cannot find one of the old repodata files make the warning look more dire make sure we look for the 'repodata' subdir inside update_md_path 2010-04-21 Seth Vidal * createrepo/__init__.py: when the update_md_path doesn't exist - emit a warning of some kind - rather than a somewhat quieter message from MetadataIndex() this is mostly to help jesse b/c he asked nicely 2010-04-16 Colin Walters * genpkgmetadata.py: if we're not a tty, don't use the progress output 2010-04-15 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Tell git to ignore tarballs. Document --repo in man page. Add bash completion. 2010-04-15 Seth Vidal * createrepo/__init__.py: - catch errors when moving the olddir out/back - if we get a yumLocalPackage object in our pkglist we should record we read it. 2010-03-05 Ville Skyttä * .gitignore: Tell git to ignore tarballs. 2010-03-05 Ville Skyttä * docs/createrepo.8: Document --repo in man page. 2010-02-19 Ville Skyttä * Makefile, createrepo.bash, createrepo.spec: Add bash completion. 2010-03-05 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Trim trailing whitespace. 2010-03-05 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: add repo tags and --repo option to describe the repo itself. request from suse. 2010-02-12 Ville Skyttä * createrepo/__init__.py, createrepo/deltarpms.py, createrepo/merge.py, createrepo/readMetadata.py, createrepo/utils.py, createrepo/yumbased.py, dmd.py, genpkgmetadata.py, mergerepo.py, modifyrepo.py: Trim trailing whitespace. 2010-02-11 Seth Vidal * genpkgmetadata.py: add option for setting max-delta-rpm-size for pkgs which are too large to be delta'd. 2010-02-10 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Make *Emacs unsuspicious about trailing whitespace. Fix --exclude -> --excludes typo. Add missing spaces in various help strings. 2010-02-10 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: add --read-pkgs-list option to output list of pkgs actually read. completely optional and only really useful with --update or a --cachedir for what pkgs DID get read/parsed. 2010-02-09 Ville Skyttä * Makefile: Make *Emacs unsuspicious about trailing whitespace. 2010-02-09 Ville Skyttä * docs/createrepo.8: Fix --exclude -> --excludes typo. 2010-02-09 Ville Skyttä * genpkgmetadata.py: Add missing spaces in various help strings. 2010-02-08 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Add missing space in --checkts help string. Ignore *.py[co]. Remove outdated comment about --baseurl. 2010-02-08 Seth Vidal * createrepo/__init__.py, docs/createrepo.8, genpkgmetadata.py: - make --unique-md-filenames the default - add simple-md-filenames to be able to disable the above if desired 2010-01-18 Ville Skyttä * genpkgmetadata.py: Add missing space in --checkts help string. 2009-09-25 Ville Skyttä * .gitignore: Ignore *.py[co]. 2009-01-26 Ville Skyttä * docs/createrepo.8: Remove outdated comment about --baseurl. At least yum uses it nowadays. 2010-01-07 Dennis Gregorovic * createrepo/readMetadata.py: Fixed, convert stat mtime to int so comparison can work, --update, BZ 553030 2010-01-07 Dennis Gregorovic * createrepo/readMetadata.py: Convert stat mtime to int so comparison can work, --update, BZ 553030 2010-01-06 Dennis Gregorovic * createrepo/__init__.py: Change baseurl of "old" packages on update, when baseurl specified 2009-10-05 Seth Vidal * createrepo/__init__.py: apply fix for https://bugzilla.redhat.com/show_bug.cgi?id=527259 make sure we're not only testing the first pkg. Test all of them until we find one that is newer. 2009-09-14 Seth Vidal * createrepo.spec: add requirement on python-deltarpm for new patch from Bill. 2009-09-14 Bill Nottingham * createrepo/deltarpms.py: createrepo patch to use the new deltarpm python bindings 2009-08-28 Seth Vidal * ChangeLog: changelog merge 2009-08-28 Seth Vidal * createrepo.spec, createrepo/__init__.py: mark as 0.9.8 2009-08-28 Seth Vidal * docs/createrepo.8: add man page entry for -n/--includepkg 2009-08-25 Seth Vidal * genpkgmetadata.py: add -n, --includepkg option to allow users to specify urls to pkgs to include in the repo on the cli - like a fully cli version of --pkglist/-i 2009-08-25 Seth Vidal * createrepo/__init__.py: capture ioerror/oserrors on handling metadata files and emit a proper MDError fixes rh bug: https://bugzilla.redhat.com/show_bug.cgi?id=514995 2009-08-18 Seth Vidal * createrepo/__init__.py: commit obviously broken pragma setting fix from mikeb@redhat.com 2009-07-21 Seth Vidal * createrepo/__init__.py: fix for https://bugzilla.redhat.com/show_bug.cgi?id=512610 take timestamp of repomd.xml - not of repodata dir - just in case repodata dir is empty, for some bizarre reason 2009-06-17 Seth Vidal * createrepo/__init__.py: remove extra 0 from max_delta_rpm_size 2009-06-17 Seth Vidal * createrepo/__init__.py: more/better output about makedeltarpm timing 2009-06-17 Seth Vidal * createrepo/__init__.py: output how long it took to make the drpm file 2009-06-16 Seth Vidal * createrepo/__init__.py: - prestodelta.xml file generation is now roughly 60X faster than it was before - python unicode string concatenation sucks, a lot. - add a delta xml generation profile output - get rid of some incorrect output about db files and delta metadata - get rid of some old not-useful comments in the code 2009-05-14 James Antill * genpkgmetadata.py: Make the UI for --checksum a bit nicer 2009-05-14 James Antill * docs/createrepo.8: Fix -profile in man page, to be --profile 2009-05-14 James Antill * docs/createrepo.8: Add some more documentation about --checksum 2009-05-13 James Antill * createrepo/__init__.py: Add open-size and size fo *_db MD. Fix file to stat for *.xml.gz size 2009-05-13 James Antill Merge branch 'master' of ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo: if our deltarpm dir doesn't exist, don't go looking for it - and definitely 2009-05-08 Seth Vidal * createrepo/__init__.py: if our deltarpm dir doesn't exist, don't go looking for it - and definitely don't traceback 2009-05-05 James Antill Merge branch 'size-in-repomd.xml' * size-in-repomd.xml: Add size to the repomd.xml output 2009-04-24 Seth Vidal * createrepo/__init__.py: pylint fixes for __init__ - lots of line cleanups and a couple of potential bugs. 2009-04-24 Seth Vidal * modifyrepo.py: pylint clean up on modifyrepo 2009-04-24 Seth Vidal * genpkgmetadata.py: genpkgmetadata.py pylint cleanup. 2009-04-22 Seth Vidal * createrepo/__init__.py: if we've not enabled the deltas, don't try to do them. silly, but harmless in this case 2009-04-21 Tim Lauridsen * createrepo/__init__.py: pylint: fixed Uses of a deprecated module 'string' 2009-04-21 Tim Lauridsen * createrepo/__init__.py, createrepo/utils.py: pylint: fixed Redefining built-in 2009-04-21 Tim Lauridsen * createrepo/deltarpms.py, createrepo/yumbased.py: pylint: fixed unused imports 2009-04-21 Tim Lauridsen * createrepo/readMetadata.py: pylint: fixed Bad indentation 2009-04-21 Tim Lauridsen * Makefile: Added the pylint basic and disabled the warning we dont care about 2009-04-18 James Antill * createrepo/__init__.py: Fix copy and paste error on message 2009-04-17 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py: make sure our sumtype specified propagates down to the pkg checksums, too 2009-04-17 Seth Vidal * createrepo/__init__.py: set a max size option so we don't kill systems with < memory than deltarpm likes to use. 2009-04-17 James Antill * createrepo/yumbased.py: Use the same checksum type for the key, as for the data in the key 2009-04-16 Seth Vidal * genpkgmetadata.py: remove the deprecation notice since: 1. it works fine 2. there is a legit use for it 2009-04-16 Seth Vidal * docs/createrepo.8: document the deltarpm options 2009-04-15 Seth Vidal * createrepo/__init__.py: it helps to have the right order of items in the pkgtup :( 2009-04-15 Seth Vidal * createrepo/__init__.py: quiet down output 2009-04-15 Seth Vidal * createrepo/__init__.py: make sure we don't try to sqlite the prestodelta xml, yet. 2009-04-15 Seth Vidal * createrepo/__init__.py: add missing '>' 2009-04-13 Seth Vidal * createrepo/__init__.py, createrepo/utils.py, modifyrepo.py: make sure the checksum type we use is being used everywhere. closes rhbug: https://bugzilla.redhat.com/show_bug.cgi?id=494951 2009-03-24 Seth Vidal * ChangeLog: changelog merge 2009-03-24 Seth Vidal * createrepo.spec, createrepo/__init__.py: 0.9.7 require yum 3.2.22 2009-02-09 Seth Vidal * createrepo/__init__.py, createrepo/deltarpms.py: when we process the rpms only do the drpm creation. after we're done take the drpms and generate the metadata from there 2009-02-03 Seth Vidal * createrepo/merge.py: some fixes and to make it work on rhel5/python2.4 2009-02-03 Seth Vidal * createrepo/__init__.py: and one more mistake 2009-02-03 Seth Vidal * createrepo/__init__.py: correct tabbing so createrepo works when you're NOT using deltas 2009-01-29 Seth Vidal * genpkgmetadata.py: add --num-deltas option 2009-01-29 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Add missing documentation on --checksum and --profile 2009-01-29 Seth Vidal * createrepo/__init__.py, createrepo/deltarpms.py, genpkgmetadata.py: --deltas, enable the creation and metadata-creation for presto/deltarpms 2009-01-27 James Antill Merge branch 'master' of ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://yum.baseurl.org/srv/projects/createrepo/git/createrepo: make modifyrepo behave with sha256 as the default checksum 2009-01-27 James Antill * docs/createrepo.8: Add missing documentation on --checksum and --profile 2009-01-27 Seth Vidal * createrepo/utils.py, modifyrepo.py: make modifyrepo behave with sha256 as the default checksum 2009-01-26 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: make sha256 the default checksum type everywhere 2009-01-23 Seth Vidal * createrepo/merge.py: add init options to specify your own yumbase object, mdconf object md generator class 2009-01-22 Seth Vidal * createrepo/utils.py: make sure we keep working on python 2.4 :( 2009-01-22 Seth Vidal * createrepo/utils.py: use gzip.name not gzip.filename to avoid python 2.6 deprecation warnings 2009-01-22 Seth Vidal * createrepo/yumbased.py: get rid of the md5 badness - use yum's Checksum class so we don't have to deal with python 2.4 vs 2.6isms 2009-01-22 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: add --profile option to the cli interface so profile info is outputted only when it is used. 2009-01-19 James Antill * createrepo/yumbased.py: Use correct cachedir after rename 2008-12-17 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: allow alternative path for --update via --update-md-path, So your old repodata does not have to be in the path you want to look through. 2008-10-28 Seth Vidal * modifyrepo.py: try/excepts on modifyrepo so we don't smack the user with a traceback 2008-10-28 Seth Vidal * ChangeLog: remerge changelog 2008-10-28 Seth Vidal * bin/Makefile: minor changes to the make file so that it will make a proper archive :) 2008-10-27 Seth Vidal * ChangeLog: merge changelog 2008-10-23 Seth Vidal * modifyrepo.py: allow already-compressed metadata files to work and not be double-compressed 2008-10-21 Seth Vidal * createrepo.spec, docs/Makefile, docs/mergerepo.1, mergerepo.py: mergerepo man page todos added to mergerepo 2008-10-21 Seth Vidal Merge branch 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo * 'master' of ssh://createrepo.baseurl.org/srv/projects/createrepo/git/createrepo: Change the NamedTemporaryFile() usage to mkstemp(), stupid API Fix parallel updates to the cachedir, thx to Michael Schwendt for spotting it 2008-10-21 Seth Vidal * AUTHORS, README, createrepo.spec, createrepo/__init__.py, docs/createrepo.8, genpkgmetadata.py: - document --content, --distro and --revision - update urls in spec and docs - add Authors file 2008-10-17 Seth Vidal * Makefile: add merge repo here, too 2008-10-17 Seth Vidal * mergerepo.py: pylintian cleanups 2008-10-17 Seth Vidal * createrepo.spec, createrepo/__init__.py: bump version to 0.9.6 - change the dep to yum 3.2.20 - since it is what WILL be needed 2008-10-17 Seth Vidal * bin/Makefile, bin/mergerepo, createrepo/merge.py, mergerepo.py: add mergerepo 2008-10-17 Seth Vidal * createrepo/__init__.py: add arbitrary metadata to config options for api callers 2008-10-09 James Antill * createrepo/yumbased.py: Change the NamedTemporaryFile() usage to mkstemp(), stupid API 2008-10-08 James Antill * createrepo/yumbased.py: Fix parallel updates to the cachedir, thx to Michael Schwendt for spotting it 2008-09-15 Seth Vidal * createrepo.spec: bump the yum requirement to 3.2.19.. 2008-09-15 Seth Vidal * createrepo/utils.py: remove unused utf8String function from utils - move most of it into yum.misc 2008-09-12 Seth Vidal * genpkgmetadata.py: make the profile option work again 2008-08-13 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: remove most of the yumbased code, disable database-only for now 2008-08-08 James Antill * createrepo/__init__.py: Add size to the repomd.xml output 2008-08-08 Seth Vidal * createrepo/__init__.py, createrepo/utils.py, createrepo/yumbased.py: minor changes for handling a packagesack and/or list of package objects as the pkglist to create the repo from also rename the xml_dump functions - eventually going to remove them. 2008-06-17 James Antill * docs/createrepo.8: Add missing doc. for --skip-stat option 2008-06-05 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: some fixmes and starts - and recommit a working --database-only 2008-05-12 James Antill * createrepo/__init__.py, genpkgmetadata.py: Remove -n option, it's a noop atm. anyway 2008-05-12 James Antill * createrepo/__init__.py: Pass just dir. to getFileList(), makes -C work. Fixes bug#446040 2008-04-16 James Antill * createrepo/utils.py: Talk to libxml maintainer ... tweak 2008-04-16 James Antill * createrepo/utils.py: Just remove bad small bytes, like 0x01 atm. 2008-04-02 James Antill * docs/createrepo.8: Add some missing options to man page 2008-03-11 Seth Vidal * createrepo/__init__.py, createrepo/utils.py: a few tweaks to speed up the database creation 2008-03-11 Seth Vidal * createrepo/__init__.py, createrepo/utils.py, createrepo/yumbased.py, docs/createrepo.8: more or less complete createrepo --database-only 2008-03-11 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: - partial patch to enable --database-only output from createrepo - still to implement filelists direct to sqlite, changelogs direct to sqlite - this check in is just a hedge against loss from my laptop, do not use the feature in this commit 2008-03-03 Seth Vidal * createrepo/__init__.py: better name for node 2008-03-03 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py: exclude rpmlib requires from metadata b/c they are silly store them in the repomd.xml per-repo so we have them if we ever actually need them 2008-02-29 James Antill * genpkgmetadata.py: Fix line overflow, minor IO optimisation 2008-02-20 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: allow --pkglist or self.conf.pkglist to be a list of arbitrary urls to packages. createrepo will download the remote urls to a tempdir, read them in and add them to the metadata. 2008-02-18 Seth Vidal * ChangeLog: merge changelog 2008-02-18 Seth Vidal * createrepo.spec, createrepo/__init__.py: bump version numbers 2008-02-18 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py: - clean up some garbage spaces and an extra 'return' - write some notes on something interesting to do for completely arbitrary repositories - make sure that under no circumstances will a package that we cannot get a pkgid/checksum from will be in the metadata. And it will output an error message 2008-02-13 Seth Vidal * createrepo/__init__.py: raise, don't print 2008-02-12 Luke Macken * genpkgmetadata.py: Clean up some more unused modules 2008-02-12 Luke Macken * createrepo/__init__.py, createrepo/utils.py: If we want to use MDError in utils.py, we need to define it outside of __init__ to avoid circular deps 2008-02-12 Luke Macken * createrepo/__init__.py: Pull in createrepo.utils.errorprint in our __init__ module. 2008-02-12 Luke Macken * createrepo/__init__.py: Import shutil since we use it in createrepo.__init__ 2008-02-12 Luke Macken * createrepo/__init__.py: s/conf.checkts/self.conf.checkts/ 2008-02-12 Luke Macken * createrepo/__init__.py, createrepo/readMetadata.py, createrepo/yumbased.py: Remove a bunch of module imports that we aren't using. One of which being 'hashlib', which prevents createrepo from running on anything by Python 2.5. 2008-01-31 Seth Vidal * createrepo/__init__.py, createrepo/readMetadata.py, createrepo/utils.py, modifyrepo.py: - make sure group files are compressed/sha-named - add group_gz section for compressed group file - add addArbitraryMetadata() method to MetaDataGenerator class - fix up modifyrepo to generate sha-named files - make modifyrepo act a bit more like createrepo for its operations 2008-01-30 Seth Vidal * ChangeLog: changelog merge 2008-01-29 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: make sure things work out as the right default 2008-01-28 Seth Vidal * createrepo.spec, createrepo/__init__.py: bump ver to 0.9.4 in spec and module 2008-01-28 Seth Vidal * createrepo/__init__.py: make sure non-unique-md-filenanmes-repos cleanup sqlite files if we switch to unique-md-filenames 2008-01-28 Seth Vidal * createrepo/__init__.py: clean up old versions of primary, filelists and other that are lingering in the repodata dir due to sha1-addition 2008-01-28 Seth Vidal * createrepo/__init__.py: swap around the filename creation order so it doesn't make leaves files around 2008-01-28 Seth Vidal * createrepo/__init__.py, createrepo/readMetadata.py: make readMetadata.py take its metadata file locations from repomd.xml using the yum parser for it. complete --unique-md-filenames implementation 2008-01-28 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: part of --unique-md-filenames is complete. This lets createrepo generate metadata files with the file's checksum included in the filename. This helps it work more nicely with proxies. --update support will not work with --unique-md-filenames at the moment. Need to read in repomd.xml to make that work. 2008-01-22 Seth Vidal * ChangeLog: changelog merge 2008-01-22 Seth Vidal * createrepo.spec, createrepo/__init__.py: 0.9.3 2008-01-22 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py, genpkgmetadata.py: add changelog-limit option to restrict the number of changelogs we add, by default 2008-01-17 Seth Vidal * genpkgmetadata.py: make sure empty directories still work 2008-01-17 Seth Vidal * ChangeLog: update changelog 2008-01-17 Seth Vidal * createrepo/__init__.py: comment out a debug print 2008-01-17 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py: re-enable --cachedir in code fix logic issues in cachedir from 0.4.X 2008-01-17 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: - fix more problems with relative paths and --split - revert cachedir disabling - cachedir isn't working yet, but it's no longer deprecated due to a use case I hadn't considered. 2008-01-17 Seth Vidal * createrepo/__init__.py: make --update and --split mostly work again 2008-01-17 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: - move a bunch of tests into the base class - deprecate --cachedir - make it enable --update instead b/c it is MUCH faster 2008-01-17 Seth Vidal * createrepo.spec, createrepo/__init__.py: bump version to 0.9.2 2008-01-17 Seth Vidal * createrepo/__init__.py: add some more correct outputs about the sqlite db generation 2008-01-17 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: --split works again, fix relative paths with ../../ curse in the direction of --split and --basedir :( 2008-01-16 Seth Vidal * createrepo/__init__.py, createrepo/utils.py, createrepo/yumbased.py, genpkgmetadata.py: clean up api to simplify all of the code calling it probably severely break --split for the moment, though 2008-01-15 Seth Vidal * createrepo/yumbased.py: make sure we have empty stubs for items in the rpm-format section, older versions of yum will go bonkers if not. 2008-01-15 Seth Vidal * createrepo/__init__.py: - add debug output for database time generation 2008-01-15 Seth Vidal * createrepo/__init__.py, createrepo/readMetadata.py, genpkgmetadata.py: - add --skip-stat to skip the stat() call on updates - this is mostly b/c the fedora createrepo call is done over nfs and that is VERY VERY slow for 20K stat() calls :( - fix --pkglist - patch from jesse keating - document options in --help output 2008-01-14 Seth Vidal * createrepo/__init__.py, createrepo/yumbased.py: - add copyright statements and licenses to top of files - start function to remove all the directory mauling in genpkgmetadata - fixmes 2008-01-11 Seth Vidal * createrepo/yumbased.py: return src for arch when it's a srpm 2008-01-10 Seth Vidal * createrepo/yumbased.py: make sure that files are run through the xml escaping, too. 2008-01-09 Seth Vidal * ChangeLog, Makefile: update changelog, add changelog target to makefile 2008-01-09 Seth Vidal * genpkgmetadata.py: clean up old comments and cruft 2008-01-09 Seth Vidal * createrepo/__init__.py, createrepo/readMetadata.py, genpkgmetadata.py: - clean up interface a lot - add callback interface for progress output - more proper catching of exceptions - remove improper sys.exit() calls from the base class 2008-01-09 Seth Vidal * createrepo/yumbased.py: make --update work correctly - by getting the right item from os.stat() for mtime 2008-01-09 Seth Vidal * createrepo/yumbased.py: free up memory in the changelog output used by generating the xml node 2008-01-08 Seth Vidal * createrepo/__init__.py: clean up a debug output 2008-01-08 Seth Vidal * createrepo/__init__.py, createrepo/readMetadata.py, createrepo/yumbased.py: fix up a lot of xml creation errors and make --update work again 2008-01-08 Seth Vidal * createrepo.spec, createrepo/__init__.py: bump to 0.9.1 2008-01-08 Seth Vidal * createrepo/yumbased.py: and a little more utf8'ing - just for completeness and insanity 2008-01-08 Seth Vidal * createrepo/yumbased.py: utf8 files, too :( 2008-01-08 Seth Vidal * createrepo/yumbased.py: all of the xml tools suck, this is evidence of them fall back to the generating the xml via libxml2 for the changelog hopefully this won't make memory explode 2008-01-08 Seth Vidal * createrepo/__init__.py: politely bounce over damaged rpms - output an error report new errorlog() method - can be easily replaced in subclass 2008-01-08 Seth Vidal * createrepo/yumbased.py: remove debug prints :) 2008-01-08 Seth Vidal * createrepo/yumbased.py: make sure that we check for nonexistent items in the hdr 2008-01-08 Seth Vidal * createrepo/__init__.py: make it a more proper ts 2008-01-08 Seth Vidal * createrepo/__init__.py: try except on package opening 2008-01-07 Seth Vidal * createrepo/yumbased.py, genpkgmetadata.py: - clean out old classes from yumbased.py - clean out debug prints from genpkgmetadata.py 2008-01-07 Seth Vidal * genpkgmetadata.py: make the version stuff make sense 2008-01-07 Seth Vidal * createrepo.spec, genpkgmetadata.py: - make rpmbuild work - mark a fixme 2008-01-03 Seth Vidal * createrepo/__init__.py, createrepo/utils.py, createrepo/yumbased.py, genpkgmetadata.py: - port to optionparser from getopt - redo config class to make use outside of cli more do-able - handle repomd.xml creation in class, too - still have a lot of changes to complete 2007-12-20 Seth Vidal * createrepo/__init__.py, genpkgmetadata.py: a little more class-full 2007-12-20 Seth Vidal * Makefile, bin/Makefile, createrepo.spec, createrepo/Makefile, createrepo/__init__.py, createrepo/readMetadata.py, createrepo/utils.py, createrepo/yumbased.py, docs/Makefile, dumpMetadata.py, genpkgmetadata.py, readMetadata.py: Whew: this is the beginning of a big conversion of createrepo to use the yum modules, behave more like a modular program and have a proper class structure. It's not done, but it's a start. 2007-12-18 Seth Vidal Merge branch 'master' of ssh://login.linux.duke.edu/home/groups/createrepo/git/createrepo * 'master' of ssh://login.linux.duke.edu/home/groups/createrepo/git/createrepo: Update ChangeLog Remove some unnecessary imports Better unicode handling in modifyrepo Add a man page for modifyrepo 2007-12-18 Seth Vidal * docs/createrepo.8, dumpMetadata.py, genpkgmetadata.py, modifyrepo.py: merge maintenance changes up to head before we nuke it from orbit 2007-12-06 Luke Macken * ChangeLog: Update ChangeLog 2007-12-06 Luke Macken * dumpMetadata.py, readMetadata.py: Remove some unnecessary imports 2007-12-06 Luke Macken * modifyrepo.py: Better unicode handling in modifyrepo 2007-12-03 Luke Macken * ChangeLog, createrepo.spec, docs/Makefile, docs/modifyrepo.1, modifyrepo.py: Add a man page for modifyrepo 2007-11-14 Seth Vidal * genpkgmetadata.py: merge pkglist option to HEAD 2007-08-08 Seth Vidal * README: update readme, point to better url, clean up explanation 2007-07-01 James Bowes * dmd.py: Add delta metadata diff and patch script 2007-06-07 Paul Nasrat * createrepo.spec: Bump version 2007-06-07 Paul Nasrat * ChangeLog, Makefile: Prepare for release 2007-06-07 Paul Nasrat * Makefile, docs/createrepo.8, genpkgmetadata.py, readMetadata.py: This patch adds a --update option to createrepo. https://lists.dulug.duke.edu/pipermail/rpm-metadata/2007-March/000756.html Patch from Mike Bonnet 2007-05-18 Paul Nasrat * dumpMetadata.py: Fix for older rpm versions Christoph Thiel 2007-05-16 Paul Nasrat * ChangeLog, Makefile, createrepo.spec, genpkgmetadata.py: Update ChangeLog Bump version to 0.4.9 2007-05-16 Paul Nasrat * dumpMetadata.py: Figure out appropriate dbversion Jeremy Katz 2007-05-16 Paul Nasrat * dumpMetadata.py: createrepo-0.4.8-cachefix.patch * changes the way the hashkey for the cache is generated. (The original version just used rpm.RPMTAG_SIGMD5, which was the same for the same signed and unsigned package. However, this lead to a wrong checksum ending up in the metadata.) Christoph Thiel 2007-05-16 Paul Nasrat * genpkgmetadata.py: createrepo-0.4.8-skip-symlinks.patch * adds an option to skip symlinks (-S, --skip-symlinks) Christoph Thiel 2007-02-13 Seth Vidal * ChangeLog: update changelog, again 2007-02-13 Seth Vidal * Makefile: add copying and copying.lib to makefile for 'make archive' 2007-02-13 Seth Vidal * ChangeLog: check in changelog 2007-02-13 Seth Vidal * Makefile: and makefile ver 2007-02-13 Seth Vidal * createrepo.spec, genpkgmetadata.py: mark as 0.4.8 2007-02-08 Paul Nasrat * COPYING.lib, createrepo.spec: Add LGPL file 2007-02-08 Paul Nasrat * COPYING, createrepo.spec: Add COPYING 2007-02-07 Seth Vidal * dumpMetadata.py: merge in Christoph Thiel's patch to fix string conversion for odd EVR's 2007-02-07 Seth Vidal * genpkgmetadata.py: merge Jesse Keatings' patch to find groups file properly 2007-02-07 Seth Vidal * Makefile: ver number in Makefile 2007-02-06 Seth Vidal * createrepo.spec: yum-metadata-parser dep and new version number 2007-02-06 Seth Vidal * docs/createrepo.8: update docs for -d 2007-02-06 Seth Vidal * genpkgmetadata.py: 0.4.7 version number 2007-02-04 Seth Vidal * dumpMetadata.py: make database version listed in repomd 2007-02-04 Seth Vidal * dumpMetadata.py: add dbversion to sqlite metadata in repomd. 2007-02-03 Seth Vidal * dumpMetadata.py: default to max compression 2007-02-03 Seth Vidal * dumpMetadata.py: make the sqlite file names not look stupid 2007-02-03 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: allow optionally creating compressed sqlite databases 2006-10-22 Luke Macken * ChangeLog, modifyrepo.py: use the mdname for the 'href' element, so it doesn't explode when dealing with xml.dom.minidom.Document objects. 2006-10-14 Luke Macken * ChangeLog, Makefile, bin/Makefile, createrepo.spec: 2006-10-14 01:30 lmacken * Makefile, bin/Makefile, createrepo.spec: Makefile changes for modifyrepo, and added it to the spec as well. 2006-08-23 Luke Macken * ChangeLog, bin/modifyrepo, modifyrepo.py: 2006-08-23 15:40 lmacken * modifyrepo.py, bin/modifyrepo: Initial import 2006-08-11 Paul Nasrat * ChangeLog: Update changelog with cvs2cl 2006-08-11 Paul Nasrat * createrepo.spec: update date 2006-08-11 Paul Nasrat * docs/createrepo.8, genpkgmetadata.py: Patch from Hans-Peter Jansen -C, --checkts option added to avoid metadata generation, if ctime filestamps are up to date. It's currently mutually exclusive with the --split option. 2006-07-28 Paul Nasrat * genpkgmetadata.py: Fix cache output dir to 0.4.5 behaviour 2006-07-28 Paul Nasrat * genpkgmetadata.py: Fix filtering out path from file list and passing correct path to writeMetaData 2006-07-28 Paul Nasrat * test/testMetaDataGenerator.py, test/testSplitMetaDataGenerator.py: nuke tests for now 2006-07-21 Paul Nasrat * Makefile: Bump 2006-07-20 Paul Nasrat * genpkgmetadata.py: Make splitmetadata handler do it' own getFileList to correctly manipulate paths. 2006-07-20 Paul Nasrat * test/testSplitMetaDataGenerator.py: Improve tests for split cases 2006-07-20 Paul Nasrat * test/testSplitMetaDataGenerator.py: duplicate for split tests 2006-07-20 Paul Nasrat * test/testMetaDataGenerator.py: More consistent naming Relative and parallel dir testing 2006-07-20 Paul Nasrat * test/testMetaDataGenerator.py: Refactor tests, add additional tests 2006-07-20 Paul Nasrat * test/testMetaDataGenerator.py: Start unit testing so we don't regress behaviour 2006-07-20 Paul Nasrat * genpkgmetadata.py: Set outputdir correctly 2006-07-20 Paul Nasrat * genpkgmetadata.py: Move to split basedir and directory everywhere to preserve command line paths. Use os.path.walk rather than our own implementation Improve error messages 2006-07-19 Paul Nasrat * createrepo.spec: genpkgmetadata.py 2006-07-19 Paul Nasrat * genpkgmetadata.py: Consistent directory handling and errors 2006-07-19 Paul Nasrat * dumpMetadata.py: Patch from hpj@urpla.net to use a more robust rpm header signature retrieval method for cache files, as recommended by Jeff Johnson. 2006-07-19 Luke Macken * ChangeLog, createrepo.spec: 2006-07-19 14:23 lmacken * createrepo.spec: remove python-urlgrabber dependency 2006-07-19 Paul Nasrat * genpkgmetadata.py: Tolerate unknown files in repodata dirs - Ville Skyttä 2006-07-19 Paul Nasrat * genpkgmetadata.py: fix up relative paths (#199228) 2006-06-30 Paul Nasrat * dumpMetadata.py: Fix srpm detection for rpm-4.4.6 and later 2006-06-26 Seth Vidal * ChangeLog: overwrite changelog 2006-06-15 Luke Macken * ChangeLog, docs/createrepo.8, genpkgmetadata.py: 2006-06-15 11:40 lmacken * genpkgmetadata.py, docs/createrepo.8: Revert --update-info-location patch, since yum now supports arbitrary metadata via YumRepository::retrieveMD() 2006-06-09 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: include Panu's patch to support --noepoch for use with old versions of rpm 2006-06-09 Seth Vidal * createrepo.spec: fix the dep 2006-06-09 Seth Vidal * createrepo.spec, genpkgmetadata.py: fix versions and bump by one. Thanks to Gareth Armstrong for noticing this. 2006-03-04 Paul Nasrat * ChangeLog: add changelog 2006-03-04 Paul Nasrat * createrepo.spec: release 2006-02-21 Paul Nasrat * Makefile, createrepo.spec, docs/createrepo.8, genpkgmetadata.py: Documentation and version updates 2006-02-21 Paul Nasrat * dumpMetadata.py, genpkgmetadata.py: Enable seperate outputdir (dgregor) 2006-02-18 Luke Macken * docs/createrepo.8, genpkgmetadata.py: Add support for -U (--update-info-location) flag to query a specified server for package update metadata. The metadata will currently be stored in 'repodata/update-info' and each package in the primary.xml will have an tag which points to it's corresponding update information. 2006-01-13 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: apply two patches from dgregor@redhat.com - verifies that the checksum cache file is more recent than the corresponding rpm - move around cmds dict initialization to make it more consistent. 2005-12-08 Paul Nasrat * genpkgmetadata.py: Fix cachedir/groupfile handling with --basedir and using paths not relative to cwd when run without --basedir. 2005-12-08 Paul Nasrat * genpkgmetadata.py: Support --split option to label media with urls across directories. 2005-12-08 Paul Nasrat * genpkgmetadata.py: Split out processing into smaller methods. Make ts internal. Files and base/file/other data attributes. 2005-12-08 Paul Nasrat * genpkgmetadata.py: Cleanup of generator class to use cmds internally as an attribute. 2005-12-08 Paul Nasrat * genpkgmetadata.py: Initial work to form metadata generator class. 2005-11-27 Seth Vidal * dumpMetadata.py: speed up by caching file mode lookup by Dennis Gregorovic 2005-11-11 Paul Nasrat * dumpMetadata.py, genpkgmetadata.py: Enable basedir to be used - dgregor 2005-11-02 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: patch from Christoph Thiel to make it work on suse 9.3 and to allow for non absolute-path cache dirs. 2005-08-11 Seth Vidal * dumpMetadata.py: turn off all signature checking when reading in headers 2005-07-24 Seth Vidal * genpkgmetadata.py: document that the -g option is for a file relative to the directory you are creating the repository for. 2005-07-14 Seth Vidal * docs/createrepo.8: man page for cachedir 2005-07-14 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: 0.4.3 2005-07-11 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: - disable the checksum flag - default and only use sha1sum's - add in -c,--cachedir option to setup a cachedir for the cache files of the checksums of the packages. Uses name-hdrid from the package hdr as filenames. Contents of the file is a single line of the package's checksum. This dramatically speeds up rebuilding a repository's metadata b/c the checksum of the package file was the item taking the most time. 2005-05-29 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: Apply Arun Bhanu's patch to add in --quiet and --verbose options instead of just -q and -v 2005-03-30 Seth Vidal * docs/Makefile: fix mandir path for docs 2005-01-18 Seth Vidal * Makefile, docs/Makefile: fix the Makefiles, f'real 2005-01-18 Seth Vidal * docs/Makefile, docs/createrepo.8: real commit 2005-01-18 Seth Vidal * Makefile, createrepo.spec: adding man page and upating the Makefiles and specfile accordingly. Thanks Bob Kashani for the man page. 2005-01-18 Seth Vidal * dumpMetadata.py: need to seek to the beginning before doing a new read operation. 2005-01-17 Seth Vidal * Makefile, createrepo.spec: spec and Makefile to 0.4.2 2005-01-17 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: move around gzipOpen for use in another program relabel 0.4.2 2005-01-07 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: single open for all file operations. about a 30% time savings. 2004-11-02 Seth Vidal * genpkgmetadata.py: small fix for --exclude to work. -x works, but --exclude didn't, now it is fixed 2004-10-21 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: update version numbers 2004-10-21 Seth Vidal * dumpMetadata.py: problem with ghost entries not showing up in primary.xml even if they matched the regex strings. 2004-10-18 Seth Vidal * bin/createrepo: whoops! need to quote that var string 2004-10-11 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: correct problem with handling dirs with a space in the filename update version number 2004-10-04 Seth Vidal * genpkgmetadata.py: clean up argument parsing to handle --version and --help more correctly. Not quite the patch Ville Skyttä submitted. 2004-09-30 Seth Vidal * genpkgmetadata.py: one more place to tag 2004-09-30 Seth Vidal * Makefile, createrepo.spec: update to 0.3.9 2004-09-30 Seth Vidal * dumpMetadata.py: checksum of group file will be wrong if specified - didn't seek(0) after copying it. 2004-09-20 Seth Vidal * genpkgmetadata.py: made 'cannot remove old metadata dir' a non-fatal error. It just warns. 2004-09-20 Seth Vidal * genpkgmetadata.py: updated to default to sha-1 checksums 2004-09-11 Seth Vidal * createrepo.spec, genpkgmetadata.py: update spec file as 0.3.8 fix for bug in command handling of groups location 2004-09-11 Seth Vidal * Makefile: fix for group file path being wrong - Bill Nottingham mark as 0.3.8 2004-09-11 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: fix for error when string is None for utf8 conversion 2004-09-03 Seth Vidal * Makefile: Makefile update to fix a bug reported by Anvil 2004-09-01 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: 0.3.7 2004-08-27 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: patch from Ville Skytta (this a will be wrong, sorry) to correct decoding/encoding problems. 2004-07-23 Seth Vidal * README: updated readme with anoncvs location 2004-07-23 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: ver to 0.3.6 2004-07-23 Seth Vidal * dumpMetadata.py: fix filelists to be complete 2004-07-23 Seth Vidal * dumpMetadata.py: remove a debug print call 2004-07-23 Seth Vidal * Makefile, createrepo.spec: mark as 0.3.5 2004-07-23 Seth Vidal * dumpMetadata.py: fix up for broken filelists in packages 2004-07-23 Seth Vidal * genpkgmetadata.py: silly string fix 2004-07-20 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: bump number to 0.3.4 2004-07-20 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: re-enabled group files documented it 2004-06-30 Seth Vidal * dumpMetadata.py: add pre=1 to requires entries for prereq marking 2004-06-30 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: - include xmlns:rpm in main metadata tag rather than per-format node - fix output for sorta-list, sorta-string rpm header tags 2004-06-28 Seth Vidal * dumpMetadata.py: fix for namespace for license, vendor, group, buildhost and sourcerpm was None, should have been formatns (rpm namespace) 2004-06-09 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: mark as 0.3.3 2004-06-06 Seth Vidal * genpkgmetadata.py: included a not-that-terribly accurate package count 2004-06-05 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: generate uncompressed checksums a much easier way. 2004-06-05 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: revert some changes 2004-06-03 Seth Vidal * genpkgmetadata.py: fix stupid version thing 2004-06-03 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: keep checksum of uncompressed metadata files in repomd.xml 2004-06-03 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: move versioned prco from separate string to properties of the entry 2004-04-16 Seth Vidal * Makefile: fix makefile 2004-04-16 Seth Vidal * Makefile, createrepo.spec, genpkgmetadata.py: update to 0.3.2 added -p or --pretty flag to make it pretty-print the xml output not pretty printing the output makes the import a lot faster and the data smaller 2004-01-18 Seth Vidal * Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py: 1. make it actually work :) 2. bump to 0.3.1 2004-01-18 Seth Vidal * Makefile, createrepo.spec: add README for real *boggle* 2004-01-18 Seth Vidal * Makefile, README, createrepo.spec: tagged Makefile and createrepo as 0.3 Add README to both of the above 2004-01-18 Seth Vidal * genpkgmetadata.py: make metadata files be written to repodata/ 2004-01-17 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: fix bug where not all files were getting included make the directory detection more reliable 2004-01-14 Seth Vidal * Makefile, bin/Makefile, createrepo.spec, dumpMetadata.py, genpkgmetadata.py: fixed memory use problem updated spec for 0.2 fixed makefile dumbness fixed problems with broken symlinks 2004-01-13 Seth Vidal * dumpMetadata.py: catch some errors on broken symlinks 2004-01-11 Seth Vidal * Makefile, bin/Makefile, bin/createrepo, createrepo.spec, dumpMetadata.py, genpkgmetadata.py: - translation stubs - makefiles - spec file - bin wrapper 2004-01-10 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: silly updates in comments 2004-01-10 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: CVS Id Tags 2004-01-10 Seth Vidal * dumpMetadata.py: [no log message] 2004-01-10 Seth Vidal * genpkgmetadata.py: added --version and __version__ string 2004-01-10 Seth Vidal * dumpMetadata.py, genpkgmetadata.py: move two functions around to more logically arrange the repomd.xml generating function 2004-01-09 Seth Vidal * Initial revision