pgstaging.orig/0000755000000000000000000000000011310154645010666 5ustar pgstaging.orig/pgstaging/0000755000000000000000000000000011310154465012651 5ustar pgstaging.orig/pgstaging/apache_listing.py0000644000000000000000000000357311271324344016206 0ustar ## ## HTML Parser for apache listings, with basic matching ## from HTMLParser import HTMLParser class ApacheListingParser(HTMLParser): """ parses HTML listing given by Apache mod_dir """ def __init__(self, content, pattern): """ buffer is a file-like object, contains the HTML source """ self.buffering = False self.buffer = "" self.content = content self.pattern = pattern HTMLParser.__init__(self) self.current_tag = None self.current_file = None self.saw_a = False self.n_td_since_a = 0 # [('filename', 'size'), ...] self.files = [] def parse(self): for line in self.content: self.feed(line) return self.files def handle_starttag(self, tag, attrs): if tag == 'a': self.buffering = True self.saw_a = True if tag == 'td': if self.saw_a: self.n_td_since_a += 1 if self.n_td_since_a == 2: self.buffering = True self.current_tag = tag def handle_endtag(self, tag): if self.current_tag == 'a': if self.buffer.find(self.pattern) > -1: self.current_file = self.buffer self.buffering = False if self.current_tag == 'td': if self.buffering and self.current_file: self.files.append((self.current_file, self.buffer)) self.buffering = False if tag == 'tr': self.n_td_since_a = 0 self.current_file = None self.saw_a = False # clean up for next tag data self.current_tag = None self.buffer = "" def handle_data(self, data): if self.buffering: if data.strip('\r\n') != '': self.buffer += data pgstaging.orig/pgstaging/__init__.py0000644000000000000000000000005311271324344014761 0ustar """ This directory is a python package """ pgstaging.orig/pgstaging/__init__.pyc0000644000000000000000000000032111271342247015124 0ustar Jc@s dZdS(s$ This directory is a python package N(t__doc__(((s?/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/__init__.pysspgstaging.orig/pgstaging/pgbouncer.pyc0000644000000000000000000001442611271342247015364 0ustar Jc@sddkZddkZddklZlZdd dYZedjo3eddZdGeiGHHd GHeiGHndS( iN(t#CouldNotGetPgBouncerConfigExceptiontSubprocessExceptiont pgbouncercBseZdZddddZdZdZdZdZd Zd Z de d Z de d Z d ZdZdZdZRS(s? PgBouncer class to get some data out of special SHOW commands tpostgress 127.0.0.1i cCs1||_||_||_||_d|_dS(s pgbouncer instance init RN(tconffiletuserthosttporttdbname(tselfRRRR((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyt__init__s     cCspg}d|i|i|i|i|f}d}ti|}d}x|djo |i}|d7}|djo4g}|idD]}||iq~} qI|djoqIqI|idjo|ddjovg} |idD]} | | iq~ } h} d}x&| D]} | | | |<|d7}q(W|i | qIqIW|i }|S( s# get pgbouncer SHOW data s3psql -h %s -p %s -U %s %s -c "SHOW %s;" 2>/dev/nullisstupid init valuetit|it(( RRRRtostpopentreadlinetsplittstriptappendtclose(R tcommandtalldatatpsqltitouttlinet_[1]tcoltheadert_[2]tctcolstdatatktcode((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pytget_datas2"    4 $0 cCs |idS(s return stats tSTATS(R$(R ((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pytstats@scCs |idS(s return pools tpools(R$(R ((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyR'DscCs |idS(s return databases t databases(R$(R ((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyR(HscCsZddk}yti|i|i}Wn+tj o}|i|IJtdnX|S(s ssh host cat config iNsSee previous output(tsystutilstssh_catRRRtstderrR(R R)tcontentterr((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyt get_configLs  cCshddk}ddkl}|}|i|i|id|i}|i||i|S(s return a ConfigParser object iN(tStringIOi( t ConfigParsert cStringIOR0twriteR/tseektSafeConfigParsertreadfpR(R R1R0tbuftconfig((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyt parse_configWs    cCsv|djo|i}n||idjo'd||f}|id||n|o|i|SndS(s- edit config file to have a new dbname in it R(sdbname=%s port=%dN(tNoneR9toptionstsetR3(R RtpgporttconfR3tdsn((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyt add_databasees cCsp|djo|i}n||idjotd|n|id||o|i|SndS(s( edit config to delete a dbname from it R(s unable to find '%s' in pgbouncerN(R:R9R;t Exceptiont remove_optionR3(R RR>R3((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyt del_databasers cCskddkl}l}|i}|i|||dtd||f}|id|||i|S(sz edit config file to have dbname point to real_dbname, and return the temporary filename containing the new configi(tVERBOSEtTERSER3sdbname=%s port=%dR((R;RDRER9R@tFalseR<R3(R Rt real_dbnameR=RDRER>R?((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pytswitch_to_databases  cCs{ddkl}ddk}|idddd\}}ti|d}|i||i|o d |GHn|S( s" write out given config to a file i(RDNtprefixs/tmp/pgbouncer.tsuffixs.initwbs#new pgbouncer.ini generated in '%s'(R;RDttempfiletmkstempRtfdopenR3R(R R>RDRLtfdtrealnamettemp((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyR3s    cCsBddkl}d|i|i|i|i|f}ti|S(s pause given database i(RDs(psql -h %s -p %s -U %s %s -c "PAUSE %s;"(R;RDRRRRR*t run_command(R RRDR((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pytpauses"cCsBddkl}d|i|i|i|i|f}ti|S(s resume given database i(RDs2/usr/bin/psql -h %s -p %s -U %s %s -c "RESUME %s;"(R;RDRRRRR*RR(R RRDR((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pytresumes"N(t__name__t __module__t__doc__R R$R&R'R(R/R9R:tTrueR@RCRHR3RSRT(((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pyR s '        t__main__s/etc/pgbouncer/pgbouncer.inis!sudo /etc/init.d/pgbouncer/reloadtPoolssConfig:(( RR*RRRRUtpR'R/(((s@/home/dim/PostgreSQL/pgfoundry/pg_staging/pgstaging/pgbouncer.pys s    pgstaging.orig/pgstaging/commands.py0000644000000000000000000006067711303004211015025 0ustar ## ## Commands available both in command line or in console ## import os, os.path, ConfigParser import utils from staging import Staging from options import DEBUG, VERBOSE, DRY_RUN from utils import WrongNumberOfArgumentsException from utils import UnknownSectionException from utils import UnknownOptionException from utils import StagingRuntimeException from utils import UnknownCommandException # cache config = None def get_option(config, section, option, optional=False): """ if [section] has no option, try in DEFAULT """ if config.has_option(section, option): return config.get(section, option) if config.has_option('default', option): return config.get('default', option) if not optional: mesg = "Unable to read %s.%s configuration" % (section, option) raise UnknownOptionException, mesg return None def parse_config(conffile, dbname, init_staging = True, force_reload = False): """ parse given INI file, and return it all if init_staging is False, return a staging object for dbname section otherwise.""" if VERBOSE: print "Parsing configuration INI file '%s'" % conffile global config if force_reload or config is None: import sys config = ConfigParser.SafeConfigParser() try: config.read(conffile) except Exception, e: print >>sys.stderr, "Error: unable to read '%s'" % conffile print >>sys.stderr, e sys.exit(2) if not config.has_section(dbname): mesg = "Error: Please provide a [%s] section" % dbname raise UnknownSectionException, mesg if init_staging: # prepare the Staging shell object try: staging = Staging(dbname, # section get_option(config, dbname, "backup_host"), get_option(config, dbname, "backup_base_url"), get_option(config, dbname, "dumpall_url", True), get_option(config, dbname, "host"), get_option(config, dbname, "dbname"), get_option(config, dbname, "dbuser"), get_option(config, dbname, "dbowner"), get_option(config, dbname, "maintdb"), get_option(config, dbname, "db_encoding"), get_option(config, dbname, "postgres_port"), get_option(config, dbname, "postgres_major"), get_option(config, dbname, "pgbouncer_port"), get_option(config, dbname, "pgbouncer_conf"), get_option(config, dbname, "remove_dump"), get_option(config, dbname, "keep_bases"), get_option(config, dbname, "auto_switch"), get_option(config, dbname, "use_sudo"), get_option(config, dbname, "pg_restore"), get_option(config, dbname, "pg_restore_st"), get_option(config, dbname, "restore_vacuum"), get_option(config, dbname, "restore_jobs"), get_option(config, dbname, "tmpdir", True)) schemas = get_option(config, dbname, "schemas", True) if schemas: schemas = [s.strip() for s in schemas.split(',')] staging.schemas = schemas schemas_nodata = get_option(config, dbname, "schemas_nodata", True) if schemas_nodata: schemas_nodata = [s.strip() for s in schemas_nodata.split(',')] staging.schemas_nodata = schemas_nodata search_path = get_option(config, dbname, "search_path", True) if search_path: search_path = [s.strip() for s in search_path.split(',')] staging.search_path = search_path replication = get_option(config, dbname, "replication", True) if replication: try: staging.replication = ConfigParser.SafeConfigParser() staging.replication.read(replication) except Exception, e: raise Exception, "Error: unable to read '%s'" % replication # Which tmpdir to use? # prefer -t command line option over .ini setup from options import TMPDIR, DEFAULT_TMPDIR if TMPDIR is not None: staging.tmpdir = TMPDIR if staging.tmpdir is None: staging.tmpdir = DEFAULT_TMPDIR # SQL PATH sql_path = None if config.has_option(dbname, "sql_path"): sql_path = config.get(dbname, "sql_path") if sql_path: # be nice, expand ~ sql_path = os.path.join(os.path.expanduser(sql_path), dbname) if os.path.isdir(sql_path): staging.sql_path = sql_path # PITR command will need a WAL Shipping slave and a way to get # the base backup and the WAL files both_same = True # none or set pitr_basedir = get_option(config, dbname, "pitr_basedir", True) base_backup = get_option(config, dbname, "base_backup", True) wal_archive = get_option(config, dbname, "wal_archive", True) if base_backup: base_backup_cmd = config.get(dbname, "backup_command", raw = False, vars = {'path': base_backup}) if wal_archive: wal_archive_cmd = config.get(dbname, "backup_command", raw = False, vars = {'path': wal_archive}) if (base_backup is None and wal_archive is not None) \ or (base_backup is not None and wal_archive is None): raise Exception, "Please configure both base_backup " +\ "and wal_archive, or none of them" if base_backup: rbb = config.get(dbname, "backup_command", raw = True).strip() if rbb[-1] != '.' and rbb[-2:] != './': raise Exception, "base_backup must end with . or ./" if pitr_basedir: staging.pitr_basedir = pitr_basedir else: raise Exception, "Please configure pitr_basedir too." if base_backup_cmd and wal_archive_cmd: staging.base_backup_cmd = base_backup_cmd staging.wal_archive_cmd = wal_archive_cmd except Exception, e: print >>sys.stderr, "Configuration error: %s" % e from options import DEBUG if DEBUG: raise sys.exit(4) return staging else: return config def parse_args_for_dbname_and_date(args, usage): """ returns dbname, date or raise WrongNumberOfArgumentsException """ if len(args) not in (1, 2): raise WrongNumberOfArgumentsException, usage dbname = args[0] date = None if len(args) == 2: date = args[1] return dbname, date def duration_pprint(duration): """ pretty print duration (human readable information) """ if duration > 3600: h = int(duration / 3600) m = int((duration - 3600 * h) / 60) s = duration - 3600 * h - 60 * m + 0.5 return '%2dh%02dm%03.1f' % (h, m, s) elif duration > 60: m = int(duration / 60) s = duration - 60 * m return ' %02dm%06.3f' % (m, s) else: return '%6.3fs' % duration ## ## Facilities to run commands, including parsing when necessary ## def run_command(conffile, command, args): """ run given pg_staging command """ import sys from pgstaging.options import VERBOSE, DRY_RUN, DEBUG # and act accordinly if command not in exports: print >>sys.stderr, "Error: no command '%s'" % command raise UnknownCommandException try: exports[command](conffile, args) except Exception, e: print >>sys.stderr, e if DEBUG: raise raise StagingRuntimeException, e def parse_input_line_and_run_command(conffile, line): """ parse input line """ from options import DEBUG, COMMENT import shlex, sys try: if len(line) > 0 and line[0] != COMMENT: cli = shlex.split(line, COMMENT) run_command(conffile, cli[0], cli[1:]) except Exception, e: raise ## ## From now on, pgstaging commands, shared by pg_staging.py and console.py ## def init_cluster(conffile, args): """ """ usage = "init " if len(args) != 1: raise WrongNumberOfArgumentsException, "init " staging = parse_config(conffile, args[0]) staging.init_cluster() def dump(conffile, args, force = False): """ dump a database """ usage = "dump [filename]" if len(args) not in [1, 2]: raise WrongNumberOfArgumentsException, usage dbname = args[0] if len(args) == 1: import datetime filename = '%s.%s.dump' % (dbname, datetime.date.today().isoformat()) else: filename = args[1] # now load configuration and dump to filename staging = parse_config(conffile, dbname) pgdump_t = staging.dump(filename, force) if pgdump_t: print " pg_dump:", duration_pprint(pgdump_t) def redump(conffile, args): """ dump a database, overwriting the pre-existing dump file if it exists """ dump(conffile, args, True) def restore(conffile, args): """ restore a database """ usage = "restore [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) wget_t, pgrestore_t, vacuumdb_t = staging.restore() print " fetching:", duration_pprint(wget_t) print "pg_restore:", duration_pprint(pgrestore_t) print " vacuum:", duration_pprint(vacuumdb_t) def restore_from_dump(conffile, args): """ """ usage = "load " if len(args) != 2: raise WrongNumberOfArgumentsException, "load " staging = parse_config(conffile, args[0]) secs = staging.load(args[1]) print "load time:", duration_pprint(secs) def fetch_dump(conffile, args): """ [date] """ usage = "fetch " dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and fetch staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) staging.get_dump() print " timing", duration_pprint(staging.wget_timing) def purge(conffile, args): """ purge """ usage = "purge " if len(args) != 1: raise WrongNumberOfArgumentsException, usage dbname = args[0] staging = parse_config(conffile, dbname) staging.purge() def vacuumdb(conffile, args): """ vacuumdb [date]""" usage = "vacuumdb [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and fetch staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) secs = staging.vacuumdb() print "vacuum took: %s" % duration_pprint(secs) def pitr(conffile, args): """ value""" usage = "pitr value" if len(args) not in (1, 3): raise WrongNumberOfArgumentsException, usage dbname = args[0] target = None value = None if len(args) == 3: if args[1] not in ('time', 'xid'): raise WrongNumberOfArgumentsException, usage target = args[1] value = args[2] # now load configuration and fetch staging = parse_config(conffile, dbname) staging.pitr(target, value) def pre_source_extra_files(conffile, args): """ [date] """ usage = "presql [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and source extra sql files staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) for sql in staging.psql_source_files(utils.PRE_SQL): print "psql -f %s" % sql def post_source_extra_files(conffile, args): """ [date] """ usage = "postsql [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and source extra sql files staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) for sql in staging.psql_source_files(utils.POST_SQL): print "psql -f %s" % sql def list_databases(conffile, args): """ list configured databases """ config = ConfigParser.SafeConfigParser() try: config.read(conffile) except Exception, e: print >>sys.stderr, "Error: unable to read '%s'" % conffile print >>sys.stderr, e sys.exit(2) for section in config.sections(): print section def list_backups(conffile, args): """ list available backups for a given database """ if len(args) != 1: raise WrongNumberOfArgumentsException, "backups " dbname = args[0] staging = parse_config(conffile, dbname) for backup, size in staging.list_backups(): print "%6s %s " % (size, backup) def psql_connect(conffile, args): """ launch a psql connection to the given configured section """ usage = "psql [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) return staging.psql_connect() def show_setting(conffile, args): """ show given database setting current value """ usage = "show [date] " if len(args) not in (2, 3): raise WrongNumberOfArgumentsException, usage if len(args) == 2: setting = args[1] dbname, backup_date = parse_args_for_dbname_and_date(args[0:1], usage) elif len(args) == 3: setting = args[2] dbname, backup_date = parse_args_for_dbname_and_date(args[0:2], usage) staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) value = staging.show(setting) print "%25s: %s" % (setting, value) def show_dbsize(conffile, args): """ show given database size """ usage = "dbsize [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) name, size, pretty = staging.dbsize() print "%25s: %s" % (name, pretty) def show_all_dbsizes(conffile, args): """ show dbsize for all databases of a dbname section """ usage = "dbsizes [ ...]" if len(args) < 1: raise WrongNumberOfArgumentsException, usage total_size = 0 if args[0] in ('--all', '--match'): config = ConfigParser.SafeConfigParser() try: config.read(conffile) except Exception, e: print >>sys.stderr, "Error: unable to read '%s'" % conffile print >>sys.stderr, e sys.exit(2) if args[0] == '--all': args = config.sections() elif args[0] == '--match': if len(args) != 2: raise WrongNumberOfArgumentsException, usage import re regexp = re.compile(args[1]) args = [x for x in config.sections() if regexp.search(x)] for db in args: # now load configuration and restore staging = parse_config(conffile, db) print db for d, s, p in staging.dbsizes(): if s > 0: total_size += s print "%25s: %s" % (d, p) print "%-25s= %s" % ('Total', staging.pg_size_pretty(total_size)) def list_pgbouncer_databases(conffile, args): """ list configured pgbouncer databases """ usage = "pgbouncer " if len(args) != 1: raise WrongNumberOfArgumentsException, usage dbname = args[0] staging = parse_config(conffile, dbname) for name, database, host, port in staging.pgbouncer_databases(): print "%25s %25s %s:%s" % (name, database, host, port) def pause_pgbouncer_database(conffile, args): """ pause [date] (when no date given, not expanded to today) """ usage = "pause [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) if backup_date is None: dbname = staging.dbname else: staging.set_backup_date(backup_date) dbname = staging.dated_dbname staging.pgbouncer_pause(dbname) def resume_pgbouncer_database(conffile, args): """ resume [date] (when no date given, not expanded to today) """ usage = "resume [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) if backup_date is None: dbname = staging.dbname else: staging.set_backup_date(backup_date) dbname = staging.dated_dbname staging.pgbouncer_resume(dbname) def switch(conffile, args): """ switch default pgbouncer config to dbname_bdate """ usage = "switch [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) staging.switch() def drop(conffile, args): """ drop given database """ usage = "drop [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) staging.drop() def catalog(conffile, args): """ [dump] print catalog for dbname, edited for nodata tables """ # experimental devel facility, not too much error handling usage = "catalog [dump]" dbname = args[0] filename = None staging = parse_config(conffile, dbname) if len(args) > 1: filename = args[1] if filename is None or not os.path.exists(filename): staging.set_backup_date(None) filename = staging.get_dump() print filename print staging.get_catalog(filename) def triggers(conffile, args): """ [dump] print triggers procedures for dbname """ # experimental devel facility, not too much error handling usage = "triggers [dump]" dbname = args[0] filename = None staging = parse_config(conffile, dbname) if len(args) > 1: filename = args[1] if filename is None or not os.path.exists(filename): staging.set_backup_date(None) filename = staging.get_dump() print filename triggers = staging.get_triggers(filename) print "%15s %-25s %s" % ('SCHEMA', 'TRIGGER', 'FUNCTION') for s in triggers: for t in triggers[s]: for f in triggers[s][t]: print "%15s %-25s %s" % (s, t, f) def set_database_search_path(conffile, args): """ alter database set search_path """ # experimental only usage = "search_path [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) staging.set_database_search_path() def list_nodata_tables(conffile, args): """ list tables to restore without their data """ usage = "nodata [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) for t in staging.get_nodata_tables(): print '%s' % t def prepare_then_run_londiste(conffile, args): """ prepare londiste files for providers of given dbname section """ usage = "londiste [date]" dbname, backup_date = parse_args_for_dbname_and_date(args, usage) # now load configuration and restore staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) for provider, filename in staging.prepare_then_run_londiste(): print "%25s: %s" % (provider, os.path.basename(filename)) def control_service(conffile, service, action, usage, args): """ internal stuff """ dbname, backup_date = parse_args_for_dbname_and_date(args, usage) staging = parse_config(conffile, dbname) staging.set_backup_date(backup_date) staging.control_service(service, action) def service_restart(conffile, args): """ restart given service depending on its configuration and special args """ usage = "restart [date]" if len(args) < 2: raise WrongNumberOfArgumentsException, usage control_service(conffile, args[0], 'restart', usage, args[1:]) def service_stop(conffile, args): """ stop given service depending on its configuration and special args """ usage = "stop [date]" if len(args) < 2: raise WrongNumberOfArgumentsException, usage control_service(conffile, args[0], 'stop', usage, args[1:]) def service_start(conffile, args): """ start given service depending on its configuration and special args """ usage = "start [date]" if len(args) < 2: raise WrongNumberOfArgumentsException, usage control_service(conffile, args[0], 'start', usage, args[1:]) def service_status(conffile, args): """ show status of given service ...""" usage = "status [date]" if len(args) < 2: raise WrongNumberOfArgumentsException, usage control_service(conffile, args[0], 'status', usage, args[1:]) def get_config_option(conffile, args): """