pax_global_header00006660000000000000000000000064123506574400014520gustar00rootroot0000000000000052 comment=d6c9fefa319edb89c8c1d54b9b41e30ef41e29b2 loofah-0.1/000077500000000000000000000000001235065744000126305ustar00rootroot00000000000000loofah-0.1/.gitignore000066400000000000000000000000271235065744000146170ustar00rootroot00000000000000*pyc *swp *loofah*egg* loofah-0.1/README.md000066400000000000000000000014151235065744000141100ustar00rootroot00000000000000loofah is a debile helper to rebuild existing set. loofah will process json files. Some examples are provided in eg/ # Commands ## Feed the database > PYTHONPATH=. ./bin/loofah-update ## Query packages Query all packages 'Build-Depends'/'Build-Depends-Indep' on cdbs: > PYTHONPATH=. ./bin/loofah-query eg/cdbs.json ## Rebuild packages > PYTHONPATH=.:../ricky/ ./bin/loofah-rebuild eg/cdbs.json default # Configuration Edit _/etc/ricky.ini_: ``` [config] signing-key= dput-target=ursae ``` Edit _~/.dput.d/profiles/ursae.json_: ``` { "fqdn": "ursae.inria.fr", "incoming": "/UploadQueue", "login": "anonymous", "meta": "boring", "method": "ftp"} ``` # Requirements * mongodb * python-pymongo * ricky https://github.com/paultag/ricky loofah-0.1/bin/000077500000000000000000000000001235065744000134005ustar00rootroot00000000000000loofah-0.1/bin/loofah-nuke000077500000000000000000000001001235065744000155250ustar00rootroot00000000000000#!/usr/bin/env python from loofah.core import db db.jobs.drop() loofah-0.1/bin/loofah-query000077500000000000000000000005401235065744000157400ustar00rootroot00000000000000#!/usr/bin/env python from loofah.core import db from loofah.sources import Sources import json import sys meta, spec = json.load(open(sys.argv[1], 'r')) for meta in db.meta.find(meta): s = Sources(meta) print("%s/%s/%s:" % (s.dist, s.version, s.suite)) entries = s.query(spec) for x in entries: print(" %s" % (x['Package'])) loofah-0.1/bin/loofah-rebuild000077500000000000000000000012441235065744000162230ustar00rootroot00000000000000#!/usr/bin/env python from loofah.core import db from loofah.sources import Sources from ricky.utils import fetch_and_upload import json import sys meta, spec = json.load(open(sys.argv[1], 'r')) group_id = sys.argv[2] def upload(x, group_id): try: return fetch_and_upload('unstable', x['Package'], x['Version'], **{ "X-Debile-Group": group_id }) except Exception: return upload(x, group_id) for meta in db.meta.find(meta): s = Sources(meta) print("%s/%s/%s:" % (s.dist, s.version, s.suite)) for x in s.query(spec): print(" Issuing a rebuild for {Package}/{Version}".format(**x)) upload(x, group_id) loofah-0.1/bin/loofah-update000077500000000000000000000004771235065744000160660ustar00rootroot00000000000000#!/usr/bin/env python import loofah.core import sys a = sys.argv if len(a) > 1: loofah.core.VERSION_NAME = a[1] if len(a) > 2: loofah.core.SUITE = a[2] from loofah.sources import digest_sources print("Digesting from remote") sources = digest_sources() print("Saving objects") sources.save() print("Done.") loofah-0.1/debian/000077500000000000000000000000001235065744000140525ustar00rootroot00000000000000loofah-0.1/debian/changelog000066400000000000000000000002041235065744000157200ustar00rootroot00000000000000loofah (0.1-1) unstable; urgency=low * Initial release -- Clément Schreiner Sun, 15 Jun 2014 14:34:58 +0200 loofah-0.1/debian/compat000066400000000000000000000000021235065744000152500ustar00rootroot000000000000009 loofah-0.1/debian/control000066400000000000000000000017121235065744000154560ustar00rootroot00000000000000Source: loofah Maintainer: Debian Python Modules Team Uploaders: Clément Schreiner Section: python Priority: optional Build-Depends: python-setuptools (>= 0.6b3), python-all (>= 2.6.6-3), debhelper (>= 9) Standards-Version: 3.9.5 Homepage: https://github.com/paultag/loofah Vcs-Git: git@github.com:paultag/loofah.git Vcs-Browser: https://github.com/paultag/loofah Package: python-loofah Architecture: all Depends: ${misc:Depends}, ${python:Depends}, python-ricky, mongodb Description: debile helper to rebuild a set of existing packages Loofah can be used to upload a subset of the Debian archive to the Debile infrastructure for rebuilding. It syncs down the apt Sources list and places it in mongodb for fast queries. This then allows the user to manually run rebuilds on any source package that fits a set of critera (written in the form of a mongo query), which get pulled and uploaded by ricky.loofah-0.1/debian/copyright000066400000000000000000000025361235065744000160130ustar00rootroot00000000000000Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: loofah Source: https://github.com/paultag/loofah Files: * Copyright: 2013, 2014 Paul Tagliamonte 2013, 2014 Sylvestre Ledru 2014 Clément Schreiner License: Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: . The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. . THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. loofah-0.1/debian/rules000077500000000000000000000002611235065744000151310ustar00rootroot00000000000000#!/usr/bin/make -f PY2_PACKAGE_NAME = python-loofah PYVERS = $(shell pyversions -r) %: dh $@ --with python2 override_dh_auto_clean: dh_auto_clean rm -rvf ./*.egg-info loofah-0.1/debian/source/000077500000000000000000000000001235065744000153525ustar00rootroot00000000000000loofah-0.1/debian/source/format000066400000000000000000000000141235065744000165600ustar00rootroot000000000000003.0 (quilt) loofah-0.1/debian/source/options000066400000000000000000000000371235065744000167700ustar00rootroot00000000000000extend-diff-ignore="\.egg-info"loofah-0.1/debian/watch000066400000000000000000000002341235065744000151020ustar00rootroot00000000000000version=3 opts="filenamemangle=s/(?:.*)?v?(\d[\d\.]*)\.tar\.gz/loofah-$1.tar.gz/" \ https://github.com/paultag/loofah/tags (?:.*/)?v?(\d[\d\.]*)\.tar\.gz loofah-0.1/eg/000077500000000000000000000000001235065744000132235ustar00rootroot00000000000000loofah-0.1/eg/c3py.json000066400000000000000000000001771235065744000150010ustar00rootroot00000000000000[ { "version": "unstable", "suite": "main" }, { "Build-Depends": "python3-all-dev" } ] loofah-0.1/eg/cdbs.json000066400000000000000000000003021235065744000150240ustar00rootroot00000000000000[ {}, { "$or": [ { "Build-Depends": "cdbs" }, { "Build-Depends-Indep": "cdbs" } ] } ] loofah-0.1/eg/cpy.json000066400000000000000000000001761235065744000147150ustar00rootroot00000000000000[ { "version": "unstable", "suite": "main" }, { "Build-Depends": "python-all-dev" } ] loofah-0.1/eg/golang.json000066400000000000000000000003221235065744000153620ustar00rootroot00000000000000[ { "suite": "main", "version": "unstable" }, { "Build-Depends": { "$in": [ "golang", "golang-go" ] } } ] loofah-0.1/eg/haskell.json000066400000000000000000000002621235065744000155410ustar00rootroot00000000000000[ { "suite": "main", "version": "unstable" }, { "Build-Depends": { "$in": [ "ghc" ] } } ] loofah-0.1/eg/python-all.json000066400000000000000000000007701235065744000162110ustar00rootroot00000000000000[ { "suite": "main", "version": "unstable" }, { "Maintainer": "Debian Python Modules Team ", "Architecture": { "$in": [ "all", "any all" ] }, "Build-Depends": { "$in": [ "python-all-dev", "python3-all-dev", "python-all", "python3-all" ] } } ] loofah-0.1/loofah/000077500000000000000000000000001235065744000141005ustar00rootroot00000000000000loofah-0.1/loofah/__init__.py000066400000000000000000000000001235065744000161770ustar00rootroot00000000000000loofah-0.1/loofah/core.py000066400000000000000000000010671235065744000154060ustar00rootroot00000000000000# from pymongo import Connection connection = Connection('localhost', 27017) db = connection.loofah PROTOCOL = "http://" MIRROR_URL = "ftp.us.debian.org" DISTRO_NAME = "debian" VERSION_NAME = "unstable" SUITE = "main" SOURCES_URI = "{protocol}{base}/{distro}/dists/{version}/{suite}/source/Sources.gz" def _get_context(): return { "protocol": PROTOCOL, "base": MIRROR_URL, "distro": DISTRO_NAME, "version": VERSION_NAME, "suite": SUITE } def get_sources_uri(): return SOURCES_URI.format(**_get_context()) loofah-0.1/loofah/sources.py000066400000000000000000000101521235065744000161340ustar00rootroot00000000000000from loofah.utils import tmpfile from loofah.core import get_sources_uri, db, _get_context from contextlib import contextmanager import urllib2 import json import gzip @contextmanager def download_sources(): with tmpfile() as fd: f = urllib2.urlopen(get_sources_uri()) open(fd, 'w').write(f.read()) yield (fd, _get_context()) @contextmanager def process_sources(): with download_sources() as info: fd, suite = info with tmpfile() as newfd: f = gzip.open(fd, 'r') try: yield (f, suite) finally: pass f.close() def digest_sources(): sent = None with process_sources() as info: fd, suite = info sent = Sources(suite) ret = {} key = None for line in fd.readlines(): if line.strip() == "": package = PackageEntry(ret) sent.add_entry(package) ret = {} continue if line[0] == " ": if ret[key] != "": ret[key] += "\n" ret[key] += line.strip() else: line = line.strip() key, val = line.split(":", 1) key = key.strip() ret[key] = val.strip() return sent class PackageEntry(dict): def __init__(self, entry, mangle=True): if mangle: entry = self._mangle(entry) for key in entry: self[key] = entry[key] def _mangle(self, entry): for key in ['Build-Depends', 'Build-Depends-Indep']: if key in entry: entry[key] = [ x.split()[0].strip() for x in entry[key].split(",") ] if 'Uploaders' in entry: entry['Uploaders'] = [ x.strip() for x in entry['Uploaders'].split(",") ] if 'Package-List' in entry: entry['Package-List'] = [ { "name": y[0], "type": y[1], "section": y[2], "priority": y[3] } for y in [ x.split() for x in entry['Package-List'].split("\n") ] ] for key in ["Files", "Checksums-Sha256", "Checksums-Sha1"]: entry[key] = [ { "hash": y[0], "size": y[1], "file": y[2] } for y in [x.split() for x in entry[key].split("\n")] ] return entry class Sources(dict): def __init__(self, info): self.info = info self.base = "{protocol}{base}".format(**info) self.suite = info['suite'] self.dist = info['distro'] self.version = info['version'] def add_entry(self, package): key = package['Package'] self[key] = package # print "I: New package: %s" % (key) def get_table(self): base, suite, dist, version = ( self.base, self.suite, self.dist, self.version ) table = getattr(getattr(getattr(db, dist), version), suite) return table def load(self): table = self.get_table() for package in table.find(): self.add_entry(PackageEntry(package)) def save(self): base, suite, dist, version = ( self.base, self.suite, self.dist, self.version ) foo = "{dist}/{ver}/{sui}".format( dist=dist, ver=version, sui=suite ) table = self.get_table() obj = self.info obj['_id'] = foo db.meta.update({"_id": foo}, obj, True, safe=True) print("Dropping old data") table.drop() print("Table dropped, reloading") for package in self: obj = self[package] obj['_id'] = obj['Package'] table.update({"_id": obj['_id']}, obj, True, safe=True) print("Updated.") def query(self, query): table = self.get_table() for package in table.find(query, timeout=False): yield package loofah-0.1/loofah/utils.py000066400000000000000000000003021235065744000156050ustar00rootroot00000000000000from contextlib import contextmanager import tempfile import os @contextmanager def tmpfile(): lolwut, fd = tempfile.mkstemp() try: yield fd finally: os.unlink(fd) loofah-0.1/requirements.txt000066400000000000000000000000101235065744000161030ustar00rootroot00000000000000pymongo loofah-0.1/setup.py000077500000000000000000000014201235065744000143420ustar00rootroot00000000000000#!/usr/bin/env python from setuptools import setup long_description = open('README.md', 'r').read() with open('requirements.txt') as f: install_requires = [l for l in f.read().splitlines() if not l.startswith('#')] setup( name="loofah", version="0.1", packages=[ "loofah" ], author="Paul Tagliamonte", author_email="paultag@debian.org", long_description=long_description, install_requires=install_requires, scripts = ['bin/loofah-nuke', 'bin/loofah-query', 'bin/loofah-rebuild', 'bin/loofah-update'], description='debile helper to rebuild a set of existing packages', license="Expat", url="https://github.com/paultag/loofah", platforms=['any'] )