barman-2.3/0000755000076500000240000000000013153300776013360 5ustar mnenciastaff00000000000000barman-2.3/AUTHORS0000644000076500000240000000251413134472625014434 0ustar mnenciastaff00000000000000Barman Core Team (in alphabetical order): * Gabriele Bartolini (architect) * Jonathan Battiato (QA/testing) * Giulio Calacoci (developer) * Francesco Canovai (QA/testing) * Leonardo Cecchi (developer) * Gianni Ciolli (QA/testing) * Britt Cole (documentation) * Marco Nenciarini (project leader) * Rubens Souza (QA/testing) Past contributors: * Carlo Ascani * Stefano Bianucci * Giuseppe Broccolo Many thanks go to our sponsors (in alphabetical order): * 4Caast - http://4caast.morfeo-project.org/ (Founding sponsor) * Adyen - http://www.adyen.com/ * Agile Business Group - http://www.agilebg.com/ * BIJ12 - http://www.bij12.nl/ * CSI Piemonte - http://www.csipiemonte.it/ (Founding sponsor) * Ecometer - http://www.ecometer.it/ * GestionaleAuto - http://www.gestionaleauto.com/ (Founding sponsor) * Jobrapido - http://www.jobrapido.com/ * Navionics - http://www.navionics.com/ (Founding sponsor) * Sovon Vogelonderzoek Nederland - https://www.sovon.nl/ * Subito.it - http://www.subito.it/ * XCon Internet Services - http://www.xcon.it/ (Founding sponsor) barman-2.3/barman/0000755000076500000240000000000013153300776014620 5ustar mnenciastaff00000000000000barman-2.3/barman/__init__.py0000644000076500000240000000153613134472625016740 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ The main Barman module """ from __future__ import absolute_import from .version import __version__ __config__ = None __all__ = ['__version__', '__config__'] barman-2.3/barman/backup.py0000644000076500000240000011252713152223257016444 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module represents a backup. """ import datetime import logging import os import shutil from glob import glob import dateutil.parser import dateutil.tz from barman import output, xlog from barman.backup_executor import PostgresBackupExecutor, RsyncBackupExecutor from barman.compression import CompressionManager from barman.config import BackupOptions from barman.exceptions import (AbortedRetryHookScript, CompressionIncompatibility, SshCommandException, UnknownBackupIdException) from barman.hooks import HookScriptRunner, RetryHookScriptRunner from barman.infofile import BackupInfo, WalFileInfo from barman.recovery_executor import RecoveryExecutor from barman.remote_status import RemoteStatusMixin from barman.utils import fsync_dir, human_readable_timedelta, pretty_size _logger = logging.getLogger(__name__) class BackupManager(RemoteStatusMixin): """Manager of the backup archive for a server""" DEFAULT_STATUS_FILTER = (BackupInfo.DONE,) def __init__(self, server): """ Constructor """ super(BackupManager, self).__init__() self.server = server self.config = server.config self._backup_cache = None self.compression_manager = CompressionManager(self.config, server.path) self.executor = None try: if self.config.backup_method == "postgres": self.executor = PostgresBackupExecutor(self) else: self.executor = RsyncBackupExecutor(self) except SshCommandException as e: self.config.disabled = True self.config.msg_list.append(str(e).strip()) @property def mode(self): """ Property defining the BackupInfo mode content """ if self.executor: return self.executor.mode return None def get_available_backups(self, status_filter=DEFAULT_STATUS_FILTER): """ Get a list of available backups :param status_filter: default DEFAULT_STATUS_FILTER. The status of the backup list returned """ # If the filter is not a tuple, create a tuple using the filter if not isinstance(status_filter, tuple): status_filter = tuple(status_filter,) # Load the cache if necessary if self._backup_cache is None: self._load_backup_cache() # Filter the cache using the status filter tuple backups = {} for key, value in self._backup_cache.items(): if value.status in status_filter: backups[key] = value return backups def _load_backup_cache(self): """ Populate the cache of the available backups, reading information from disk. """ self._backup_cache = {} # Load all the backups from disk reading the backup.info files for filename in glob("%s/*/backup.info" % self.config.basebackups_directory): backup = BackupInfo(self.server, filename) self._backup_cache[backup.backup_id] = backup def backup_cache_add(self, backup_info): """ Register a BackupInfo object to the backup cache. NOTE: Initialise the cache - in case it has not been done yet :param barman.infofile.BackupInfo backup_info: the object we want to register in the cache """ # Load the cache if needed if self._backup_cache is None: self._load_backup_cache() # Insert the BackupInfo object into the cache self._backup_cache[backup_info.backup_id] = backup_info def backup_cache_remove(self, backup_info): """ Remove a BackupInfo object from the backup cache This method _must_ be called after removing the object from disk. :param barman.infofile.BackupInfo backup_info: the object we want to remove from the cache """ # Nothing to do if the cache is not loaded if self._backup_cache is None: return # Remove the BackupInfo object from the backups cache del self._backup_cache[backup_info.backup_id] def get_backup(self, backup_id): """ Return the backup information for the given backup id. If the backup_id is None or backup.info file doesn't exists, it returns None. :param str|None backup_id: the ID of the backup to return :rtype: BackupInfo|None """ if backup_id is not None: # Get all the available backups from the cache available_backups = self.get_available_backups( BackupInfo.STATUS_ALL) # Return the BackupInfo if present, or None return available_backups.get(backup_id) return None def get_previous_backup(self, backup_id, status_filter=DEFAULT_STATUS_FILTER): """ Get the previous backup (if any) in the catalog :param status_filter: default DEFAULT_STATUS_FILTER. The status of the backup returned """ if not isinstance(status_filter, tuple): status_filter = tuple(status_filter) backup = BackupInfo(self.server, backup_id=backup_id) available_backups = self.get_available_backups(status_filter + (backup.status,)) ids = sorted(available_backups.keys()) try: current = ids.index(backup_id) while current > 0: res = available_backups[ids[current - 1]] if res.status in status_filter: return res current -= 1 return None except ValueError: raise UnknownBackupIdException('Could not find backup_id %s' % backup_id) def get_next_backup(self, backup_id, status_filter=DEFAULT_STATUS_FILTER): """ Get the next backup (if any) in the catalog :param status_filter: default DEFAULT_STATUS_FILTER. The status of the backup returned """ if not isinstance(status_filter, tuple): status_filter = tuple(status_filter) backup = BackupInfo(self.server, backup_id=backup_id) available_backups = self.get_available_backups(status_filter + (backup.status,)) ids = sorted(available_backups.keys()) try: current = ids.index(backup_id) while current < (len(ids) - 1): res = available_backups[ids[current + 1]] if res.status in status_filter: return res current += 1 return None except ValueError: raise UnknownBackupIdException('Could not find backup_id %s' % backup_id) def get_last_backup_id(self, status_filter=DEFAULT_STATUS_FILTER): """ Get the id of the latest/last backup in the catalog (if exists) :param status_filter: The status of the backup to return, default to DEFAULT_STATUS_FILTER. :return string|None: ID of the backup """ available_backups = self.get_available_backups(status_filter) if len(available_backups) == 0: return None ids = sorted(available_backups.keys()) return ids[-1] def get_first_backup_id(self, status_filter=DEFAULT_STATUS_FILTER): """ Get the id of the oldest/first backup in the catalog (if exists) :param status_filter: The status of the backup to return, default to DEFAULT_STATUS_FILTER. :return string|None: ID of the backup """ available_backups = self.get_available_backups(status_filter) if len(available_backups) == 0: return None ids = sorted(available_backups.keys()) return ids[0] def delete_backup(self, backup): """ Delete a backup :param backup: the backup to delete """ available_backups = self.get_available_backups() minimum_redundancy = self.server.config.minimum_redundancy # Honour minimum required redundancy if backup.status == BackupInfo.DONE and \ minimum_redundancy >= len(available_backups): output.warning("Skipping delete of backup %s for server %s " "due to minimum redundancy requirements " "(minimum redundancy = %s, " "current redundancy = %s)", backup.backup_id, self.config.name, len(available_backups), minimum_redundancy) return # Keep track of when the delete operation started. delete_start_time = datetime.datetime.now() output.info("Deleting backup %s for server %s", backup.backup_id, self.config.name) previous_backup = self.get_previous_backup(backup.backup_id) next_backup = self.get_next_backup(backup.backup_id) # Delete all the data contained in the backup try: self.delete_backup_data(backup) except OSError as e: output.error("Failure deleting backup %s for server %s.\n%s", backup.backup_id, self.config.name, e) return # Check if we are deleting the first available backup if not previous_backup: # In the case of exclusive backup (default), removes any WAL # files associated to the backup being deleted. # In the case of concurrent backup, removes only WAL files # prior to the start of the backup being deleted, as they # might be useful to any concurrent backup started immediately # after. remove_until = None # means to remove all WAL files if next_backup: remove_until = next_backup elif BackupOptions.CONCURRENT_BACKUP in self.config.backup_options: remove_until = backup timelines_to_protect = set() # If remove_until is not set there are no backup left if remove_until: # Retrieve the list of extra timelines that contains at least # a backup. On such timelines we don't want to delete any WAL for value in self.get_available_backups( BackupInfo.STATUS_ARCHIVING).values(): # Ignore the backup that is being deleted if value == backup: continue timelines_to_protect.add(value.timeline) # Remove the timeline of `remove_until` from the list. # We have enough information to safely delete unused WAL files # on it. timelines_to_protect -= set([remove_until.timeline]) output.info("Delete associated WAL segments:") for name in self.remove_wal_before_backup(remove_until, timelines_to_protect): output.info("\t%s", name) # As last action, remove the backup directory, # ending the delete operation try: self.delete_basebackup(backup) except OSError as e: output.error("Failure deleting backup %s for server %s.\n%s\n" "Please manually remove the '%s' directory", backup.backup_id, self.config.name, e, backup.get_basebackup_directory()) return self.backup_cache_remove(backup) # Save the time of the complete removal of the backup delete_end_time = datetime.datetime.now() output.info("Deleted backup %s (start time: %s, elapsed time: %s)", backup.backup_id, delete_start_time.ctime(), human_readable_timedelta( delete_end_time - delete_start_time)) def backup(self): """ Performs a backup for the server """ _logger.debug("initialising backup information") self.executor.init() backup_info = None try: # Create the BackupInfo object representing the backup backup_info = BackupInfo( self.server, backup_id=datetime.datetime.now().strftime('%Y%m%dT%H%M%S')) backup_info.save() self.backup_cache_add(backup_info) output.info( "Starting backup using %s method for server %s in %s", self.mode, self.config.name, backup_info.get_basebackup_directory()) # Run the pre-backup-script if present. script = HookScriptRunner(self, 'backup_script', 'pre') script.env_from_backup_info(backup_info) script.run() # Run the pre-backup-retry-script if present. retry_script = RetryHookScriptRunner( self, 'backup_retry_script', 'pre') retry_script.env_from_backup_info(backup_info) retry_script.run() # Do the backup using the BackupExecutor self.executor.backup(backup_info) # Compute backup size and fsync it on disk self.backup_fsync_and_set_sizes(backup_info) # Mark the backup as DONE backup_info.set_attribute("status", "DONE") # Use BaseException instead of Exception to catch events like # KeyboardInterrupt (e.g.: CRTL-C) except BaseException as e: msg_lines = str(e).strip().splitlines() if backup_info: # Use only the first line of exception message # in backup_info error field backup_info.set_attribute("status", "FAILED") # If the exception has no attached message use the raw # type name if len(msg_lines) == 0: msg_lines = [type(e).__name__] backup_info.set_attribute( "error", "failure %s (%s)" % ( self.executor.current_action, msg_lines[0])) output.error("Backup failed %s.\nDETAILS: %s\n%s", self.executor.current_action, msg_lines[0], '\n'.join(msg_lines[1:])) else: output.info("Backup end at LSN: %s (%s, %08X)", backup_info.end_xlog, backup_info.end_wal, backup_info.end_offset) output.info("Backup completed (start time: %s, elapsed time: %s)", self.executor.copy_start_time, human_readable_timedelta( self.executor.copy_end_time - self.executor.copy_start_time)) # Create a restore point after a backup target_name = 'barman_%s' % backup_info.backup_id self.server.postgres.create_restore_point(target_name) finally: if backup_info: backup_info.save() # Make sure we are not holding any PostgreSQL connection # during the post-backup scripts self.server.close() # Run the post-backup-retry-script if present. try: retry_script = RetryHookScriptRunner( self, 'backup_retry_script', 'post') retry_script.env_from_backup_info(backup_info) retry_script.run() except AbortedRetryHookScript as e: # Ignore the ABORT_STOP as it is a post-hook operation _logger.warning("Ignoring stop request after receiving " "abort (exit code %d) from post-backup " "retry hook script: %s", e.hook.exit_status, e.hook.script) # Run the post-backup-script if present. script = HookScriptRunner(self, 'backup_script', 'post') script.env_from_backup_info(backup_info) script.run() output.result('backup', backup_info) def recover(self, backup_info, dest, tablespaces=None, target_tli=None, target_time=None, target_xid=None, target_name=None, target_immediate=False, exclusive=False, remote_command=None): """ Performs a recovery of a backup :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None target_tli: the target timeline :param str|None target_time: the target time :param str|None target_xid: the target xid :param str|None target_name: the target name created previously with pg_create_restore_point() function call :param bool|None target_immediate: end recovery as soon as consistency is reached :param bool exclusive: whether the recovery is exclusive or not :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. """ # Archive every WAL files in the incoming directory of the server self.server.archive_wal(verbose=False) # Delegate the recovery operation to a RecoveryExecutor object executor = RecoveryExecutor(self) recovery_info = executor.recover(backup_info, dest, tablespaces, target_tli, target_time, target_xid, target_name, target_immediate, exclusive, remote_command) # Output recovery results output.result('recovery', recovery_info['results']) def archive_wal(self, verbose=True): """ Executes WAL maintenance operations, such as archiving and compression If verbose is set to False, outputs something only if there is at least one file :param bool verbose: report even if no actions """ for archiver in self.server.archivers: archiver.archive(verbose) def cron_retention_policy(self): """ Retention policy management """ if (self.server.enforce_retention_policies and self.config.retention_policy_mode == 'auto'): available_backups = self.get_available_backups( BackupInfo.STATUS_ALL) retention_status = self.config.retention_policy.report() for bid in sorted(retention_status.keys()): if retention_status[bid] == BackupInfo.OBSOLETE: output.info( "Enforcing retention policy: removing backup %s for " "server %s" % (bid, self.config.name)) self.delete_backup(available_backups[bid]) def delete_basebackup(self, backup): """ Delete the basebackup dir of a given backup. :param barman.infofile.BackupInfo backup: the backup to delete """ backup_dir = backup.get_basebackup_directory() _logger.debug("Deleting base backup directory: %s" % backup_dir) shutil.rmtree(backup_dir) def delete_backup_data(self, backup): """ Delete the data contained in a given backup. :param barman.infofile.BackupInfo backup: the backup to delete """ if backup.tablespaces: if backup.backup_version == 2: tbs_dir = backup.get_basebackup_directory() else: tbs_dir = os.path.join(backup.get_data_directory(), 'pg_tblspc') for tablespace in backup.tablespaces: rm_dir = os.path.join(tbs_dir, str(tablespace.oid)) if os.path.exists(rm_dir): _logger.debug("Deleting tablespace %s directory: %s" % (tablespace.name, rm_dir)) shutil.rmtree(rm_dir) pg_data = backup.get_data_directory() if os.path.exists(pg_data): _logger.debug("Deleting PGDATA directory: %s" % pg_data) shutil.rmtree(pg_data) def delete_wal(self, wal_info): """ Delete a WAL segment, with the given WalFileInfo :param barman.infofile.WalFileInfo wal_info: the WAL to delete """ try: os.unlink(wal_info.fullpath(self.server)) try: os.removedirs(os.path.dirname(wal_info.fullpath(self.server))) except OSError: # This is not an error condition # We always try to remove the the trailing directories, # this means that hashdir is not empty. pass except OSError as e: output.warning('Ignoring deletion of WAL file %s ' 'for server %s: %s', wal_info.name, self.config.name, e) def check(self, check_strategy): """ This function does some checks on the server. :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('compression settings') # Check compression_setting parameter if self.config.compression and not self.compression_manager.check(): check_strategy.result(self.config.name, False) else: status = True try: self.compression_manager.get_compressor() except CompressionIncompatibility as field: check_strategy.result(self.config.name, '%s setting' % field, False) status = False check_strategy.result(self.config.name, status) # Failed backups check check_strategy.init_check('failed backups') failed_backups = self.get_available_backups((BackupInfo.FAILED,)) status = len(failed_backups) == 0 check_strategy.result( self.config.name, status, hint='there are %s failed backups' % (len(failed_backups,)) ) check_strategy.init_check('minimum redundancy requirements') # Minimum redundancy checks no_backups = len(self.get_available_backups()) # Check minimum_redundancy_requirements parameter if no_backups < int(self.config.minimum_redundancy): status = False else: status = True check_strategy.result( self.config.name, status, hint='have %s backups, expected at least %s' % ( no_backups, self.config.minimum_redundancy)) # TODO: Add a check for the existence of ssh and of rsync # Execute additional checks defined by the BackupExecutor if self.executor: self.executor.check(check_strategy) def status(self): """ This function show the server status """ # get number of backups no_backups = len(self.get_available_backups()) output.result('status', self.config.name, "backups_number", "No. of available backups", no_backups) output.result('status', self.config.name, "first_backup", "First available backup", self.get_first_backup_id()) output.result('status', self.config.name, "last_backup", "Last available backup", self.get_last_backup_id()) # Minimum redundancy check. if number of backups minor than minimum # redundancy, fail. if no_backups < self.config.minimum_redundancy: output.result('status', self.config.name, "minimum_redundancy", "Minimum redundancy requirements", "FAILED (%s/%s)" % ( no_backups, self.config.minimum_redundancy)) else: output.result('status', self.config.name, "minimum_redundancy", "Minimum redundancy requirements", "satisfied (%s/%s)" % ( no_backups, self.config.minimum_redundancy)) # Output additional status defined by the BackupExecutor if self.executor: self.executor.status() def fetch_remote_status(self): """ Build additional remote status lines defined by the BackupManager. This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ if self.executor: return self.executor.get_remote_status() else: return {} def rebuild_xlogdb(self): """ Rebuild the whole xlog database guessing it from the archive content. """ from os.path import isdir, join output.info("Rebuilding xlogdb for server %s", self.config.name) root = self.config.wals_directory default_compression = self.config.compression wal_count = label_count = history_count = 0 # lock the xlogdb as we are about replacing it completely with self.server.xlogdb('w') as fxlogdb: xlogdb_new = fxlogdb.name + ".new" with open(xlogdb_new, 'w') as fxlogdb_new: for name in sorted(os.listdir(root)): # ignore the xlogdb and its lockfile if name.startswith(self.server.XLOG_DB): continue fullname = join(root, name) if isdir(fullname): # all relevant files are in subdirectories hash_dir = fullname for wal_name in sorted(os.listdir(hash_dir)): fullname = join(hash_dir, wal_name) if isdir(fullname): _logger.warning( 'unexpected directory ' 'rebuilding the wal database: %s', fullname) else: if xlog.is_wal_file(fullname): wal_count += 1 elif xlog.is_backup_file(fullname): label_count += 1 elif fullname.endswith('.tmp'): _logger.warning( 'temporary file found ' 'rebuilding the wal database: %s', fullname) continue else: _logger.warning( 'unexpected file ' 'rebuilding the wal database: %s', fullname) continue wal_info = WalFileInfo.from_file( fullname, default_compression=default_compression) fxlogdb_new.write(wal_info.to_xlogdb_line()) else: # only history files are here if xlog.is_history_file(fullname): history_count += 1 wal_info = WalFileInfo.from_file( fullname, default_compression=default_compression) fxlogdb_new.write(wal_info.to_xlogdb_line()) else: _logger.warning( 'unexpected file ' 'rebuilding the wal database: %s', fullname) os.fsync(fxlogdb_new.fileno()) shutil.move(xlogdb_new, fxlogdb.name) fsync_dir(os.path.dirname(fxlogdb.name)) output.info('Done rebuilding xlogdb for server %s ' '(history: %s, backup_labels: %s, wal_file: %s)', self.config.name, history_count, label_count, wal_count) def get_latest_archived_wals_info(self): """ Return a dictionary of timelines associated with the WalFileInfo of the last WAL file in the archive, or None if the archive doesn't contain any WAL file. :rtype: dict[str, WalFileInfo]|None """ from os.path import isdir, join root = self.config.wals_directory # If the WAL archive directory doesn't exists the archive is empty if not isdir(root): return None # Traverse all the directory in the archive in reverse order, # returning the first WAL file found timelines = {} for name in sorted(os.listdir(root), reverse=True): fullname = join(root, name) # All relevant files are in subdirectories, so # we skip any non-directory entry if isdir(fullname): # Extract the timeline. If it is not valid, skip this directory try: timeline = name[0:8] int(timeline, 16) except ValueError: continue # If this timeline already has a file, skip this directory if timeline in timelines: continue hash_dir = fullname # Inspect contained files in reverse order for wal_name in sorted(os.listdir(hash_dir), reverse=True): fullname = join(hash_dir, wal_name) # Return the first file that has the correct name if not isdir(fullname) and xlog.is_wal_file(fullname): timelines[timeline] = WalFileInfo.from_file(fullname) break # Return the timeline map or None if it is empty return timelines or None def remove_wal_before_backup(self, backup_info, timelines_to_protect=None): """ Remove WAL files which have been archived before the start of the provided backup. If no backup_info is provided delete all available WAL files If timelines_to_protect list is passed, never remove a wal in one of these timelines. :param BackupInfo|None backup_info: the backup information structure :param set timelines_to_protect: optional list of timelines to protect :return list: a list of removed WAL files """ removed = [] with self.server.xlogdb() as fxlogdb: xlogdb_new = fxlogdb.name + ".new" with open(xlogdb_new, 'w') as fxlogdb_new: for line in fxlogdb: wal_info = WalFileInfo.from_xlogdb_line(line) if not xlog.is_any_xlog_file(wal_info.name): output.error( "invalid WAL segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", wal_info.name, self.config.name) continue # Keeps the WAL segment if it is a history file keep = xlog.is_history_file(wal_info.name) # Keeps the WAL segment if its timeline is in # `timelines_to_protect` if timelines_to_protect: tli, _, _ = xlog.decode_segment_name(wal_info.name) keep |= tli in timelines_to_protect # Keeps the WAL segment if it is a newer # than the given backup (the first available) if backup_info: keep |= wal_info.name >= backup_info.begin_wal # If the file has to be kept write it in the new xlogdb # otherwise delete it and record it in the removed list if keep: fxlogdb_new.write(wal_info.to_xlogdb_line()) else: self.delete_wal(wal_info) removed.append(wal_info.name) fxlogdb_new.flush() os.fsync(fxlogdb_new.fileno()) shutil.move(xlogdb_new, fxlogdb.name) fsync_dir(os.path.dirname(fxlogdb.name)) return removed def validate_last_backup_maximum_age(self, last_backup_maximum_age): """ Evaluate the age of the last available backup in a catalogue. If the last backup is older than the specified time interval (age), the function returns False. If within the requested age interval, the function returns True. :param timedate.timedelta last_backup_maximum_age: time interval representing the maximum allowed age for the last backup in a server catalogue :return tuple: a tuple containing the boolean result of the check and auxiliary information about the last backup current age """ # Get the ID of the last available backup backup_id = self.get_last_backup_id() if backup_id: # Get the backup object backup = BackupInfo(self.server, backup_id=backup_id) now = datetime.datetime.now(dateutil.tz.tzlocal()) # Evaluate the point of validity validity_time = now - last_backup_maximum_age # Pretty print of a time interval (age) msg = human_readable_timedelta(now - backup.end_time) # If the backup end time is older than the point of validity, # return False, otherwise return true if backup.end_time < validity_time: return False, msg else: return True, msg else: # If no backup is available return false return False, "No available backups" def backup_fsync_and_set_sizes(self, backup_info): """ Fsync all files in a backup and set the actual size on disk of a backup. Also evaluate the deduplication ratio and the deduplicated size if applicable. :param barman.infofile.BackupInfo backup_info: the backup to update """ # Calculate the base backup size self.executor.current_action = "calculating backup size" _logger.debug(self.executor.current_action) backup_size = 0 deduplicated_size = 0 backup_dest = backup_info.get_basebackup_directory() for dir_path, _, file_names in os.walk(backup_dest): # execute fsync() on the containing directory fsync_dir(dir_path) # execute fsync() on all the contained files for filename in file_names: file_path = os.path.join(dir_path, filename) file_fd = os.open(file_path, os.O_RDONLY) file_stat = os.fstat(file_fd) backup_size += file_stat.st_size # Excludes hard links from real backup size if file_stat.st_nlink == 1: deduplicated_size += file_stat.st_size os.fsync(file_fd) os.close(file_fd) # Save size into BackupInfo object backup_info.set_attribute('size', backup_size) backup_info.set_attribute('deduplicated_size', deduplicated_size) if backup_info.size > 0: deduplication_ratio = 1 - (float( backup_info.deduplicated_size) / backup_info.size) else: deduplication_ratio = 0 if self.config.reuse_backup == 'link': output.info( "Backup size: %s. Actual size on disk: %s" " (-%s deduplication ratio)." % ( pretty_size(backup_info.size), pretty_size(backup_info.deduplicated_size), '{percent:.2%}'.format(percent=deduplication_ratio) )) else: output.info("Backup size: %s" % pretty_size(backup_info.size)) barman-2.3/barman/backup_executor.py0000644000076500000240000020271313152223257020357 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ Backup Executor module A Backup Executor is a class responsible for the execution of a backup. Specific implementations of backups are defined by classes that derive from BackupExecutor (e.g.: backup with rsync through Ssh). A BackupExecutor is invoked by the BackupManager for backup operations. """ import datetime import logging import os import re import shutil from abc import ABCMeta, abstractmethod from functools import partial import dateutil.parser from distutils.version import LooseVersion as Version from barman import output, xlog from barman.command_wrappers import PgBaseBackup from barman.config import BackupOptions from barman.copy_controller import RsyncCopyController from barman.exceptions import (CommandFailedException, DataTransferFailure, FsOperationFailed, PostgresConnectionError, PostgresIsInRecovery, SshCommandException) from barman.fs import UnixRemoteCommand from barman.infofile import BackupInfo from barman.remote_status import RemoteStatusMixin from barman.utils import (human_readable_timedelta, mkpath, total_seconds, with_metaclass) _logger = logging.getLogger(__name__) class BackupExecutor(with_metaclass(ABCMeta, RemoteStatusMixin)): """ Abstract base class for any backup executors. """ def __init__(self, backup_manager, mode=None): """ Base constructor :param barman.backup.BackupManager backup_manager: the BackupManager assigned to the executor """ super(BackupExecutor, self).__init__() self.backup_manager = backup_manager self.server = backup_manager.server self.config = backup_manager.config self.strategy = None self._mode = mode self.copy_start_time = None self.copy_end_time = None # Holds the action being executed. Used for error messages. self.current_action = None def init(self): """ Initialise the internal state of the backup executor """ self.current_action = "starting backup" @property def mode(self): """ Property that defines the mode used for the backup. If a strategy is present, the returned string is a combination of the mode of the executor and the mode of the strategy (eg: rsync-exclusive) :return str: a string describing the mode used for the backup """ strategy_mode = self.strategy.mode if strategy_mode: return "%s-%s" % (self._mode, strategy_mode) else: return self._mode @abstractmethod def backup(self, backup_info): """ Perform a backup for the server - invoked by BackupManager.backup() :param barman.infofile.BackupInfo backup_info: backup information """ def check(self, check_strategy): """ Perform additional checks - invoked by BackupManager.check() :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ def status(self): """ Set additional status info - invoked by BackupManager.status() """ def fetch_remote_status(self): """ Get additional remote status info - invoked by BackupManager.get_remote_status() This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ return {} def _purge_unused_wal_files(self, backup_info): """ It the provided backup is the first, purge all WAL files before the backup start. :param barman.infofile.BackupInfo backup_info: the backup to check """ # Do nothing if the begin_wal is not defined yet if backup_info.begin_wal is None: return # If this is the first backup, purge unused WAL files previous_backup = self.backup_manager.get_previous_backup( backup_info.backup_id) if not previous_backup: output.info("This is the first backup for server %s", self.config.name) removed = self.backup_manager.remove_wal_before_backup( backup_info) if removed: # report the list of the removed WAL files output.info("WAL segments preceding the current backup " "have been found:", log=False) for wal_name in removed: output.info("\t%s from server %s " "has been removed", wal_name, self.config.name) def _start_backup_copy_message(self, backup_info): """ Output message for backup start :param barman.infofile.BackupInfo backup_info: backup information """ output.info("Copying files for %s", backup_info.backup_id) def _stop_backup_copy_message(self, backup_info): """ Output message for backup end :param barman.infofile.BackupInfo backup_info: backup information """ output.info("Copy done (time: %s)", human_readable_timedelta(datetime.timedelta( seconds=backup_info.copy_stats['copy_time']))) def _parse_ssh_command(ssh_command): """ Parse a user provided ssh command to a single command and a list of arguments In case of error, the first member of the result (the command) will be None :param ssh_command: a ssh command provided by the user :return tuple[str,list[str]]: the command and a list of options """ try: ssh_options = ssh_command.split() except AttributeError: return None, [] ssh_command = ssh_options.pop(0) ssh_options.extend("-o BatchMode=yes -o StrictHostKeyChecking=no".split()) return ssh_command, ssh_options class PostgresBackupExecutor(BackupExecutor): """ Concrete class for backup via pg_basebackup (plain format). Relies on pg_basebackup command to copy data files from the PostgreSQL cluster using replication protocol. """ def __init__(self, backup_manager): """ Constructor :param barman.backup.BackupManager backup_manager: the BackupManager assigned to the executor """ super(PostgresBackupExecutor, self).__init__(backup_manager, 'postgres') self.validate_configuration() self.strategy = PostgresBackupStrategy(self) def validate_configuration(self): """ Validate the configuration for this backup executor. If the configuration is not compatible this method will disable the server. """ # Check for the correct backup options if BackupOptions.EXCLUSIVE_BACKUP in self.config.backup_options: self.config.backup_options.remove( BackupOptions.EXCLUSIVE_BACKUP) output.warning( "'exclusive_backup' is not a valid backup_option " "using postgres backup_method. " "Overriding with 'concurrent_backup'.") # Apply the default backup strategy if BackupOptions.CONCURRENT_BACKUP not in \ self.config.backup_options: self.config.backup_options.add(BackupOptions.CONCURRENT_BACKUP) output.debug("The default backup strategy for " "postgres backup_method is: concurrent_backup") # Forbid tablespace_bandwidth_limit option. # It works only with rsync based backups. if self.config.tablespace_bandwidth_limit: self.server.config.disabled = True # Report the error in the configuration errors message list self.server.config.msg_list.append( 'tablespace_bandwidth_limit option is not supported by ' 'postgres backup_method') # Forbid reuse_backup option. # It works only with rsync based backups. if self.config.reuse_backup in ('copy', 'link'): self.server.config.disabled = True # Report the error in the configuration errors message list self.server.config.msg_list.append( 'reuse_backup option is not supported by ' 'postgres backup_method') # Forbid network_compression option. # It works only with rsync based backups. if self.config.network_compression: self.server.config.disabled = True # Report the error in the configuration errors message list self.server.config.msg_list.append( 'network_compression option is not supported by ' 'postgres backup_method') # bandwidth_limit option is supported by pg_basebackup executable # starting from Postgres 9.4 if self.server.config.bandwidth_limit: # This method is invoked too early to have a working streaming # connection. So we avoid caching the result by directly # invoking fetch_remote_status() instead of get_remote_status() remote_status = self.fetch_remote_status() # If pg_basebackup is present and it doesn't support bwlimit # disable the server. if remote_status['pg_basebackup_bwlimit'] is False: self.server.config.disabled = True # Report the error in the configuration errors message list self.server.config.msg_list.append( "bandwidth_limit option is not supported by " "pg_basebackup version (current: %s, required: 9.4)" % remote_status['pg_basebackup_version']) def backup(self, backup_info): """ Perform a backup for the server - invoked by BackupManager.backup() through the generic interface of a BackupExecutor. This implementation is responsible for performing a backup through the streaming protocol. The connection must be made with a superuser or a user having REPLICATION permissions (see PostgreSQL documentation, Section 20.2), and pg_hba.conf must explicitly permit the replication connection. The server must also be configured with enough max_wal_senders to leave at least one session available for the backup. :param barman.infofile.BackupInfo backup_info: backup information """ try: # Set data directory and server version self.strategy.start_backup(backup_info) backup_info.save() if backup_info.begin_wal is not None: output.info("Backup start at LSN: %s (%s, %08X)", backup_info.begin_xlog, backup_info.begin_wal, backup_info.begin_offset) else: output.info("Backup start at LSN: %s", backup_info.begin_xlog) # Start the copy self.current_action = "copying files" self._start_backup_copy_message(backup_info) self.backup_copy(backup_info) self._stop_backup_copy_message(backup_info) self.strategy.stop_backup(backup_info) # If this is the first backup, purge eventually unused WAL files self._purge_unused_wal_files(backup_info) except CommandFailedException as e: _logger.exception(e) raise def check(self, check_strategy): """ Perform additional checks for PostgresBackupExecutor :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('pg_basebackup') remote_status = self.get_remote_status() # Check for the presence of pg_basebackup check_strategy.result( self.config.name, remote_status['pg_basebackup_installed']) # remote_status['pg_basebackup_compatible'] is None if # pg_basebackup cannot be executed and False if it is # not compatible. hint = None check_strategy.init_check('pg_basebackup compatible') if not remote_status['pg_basebackup_compatible']: pg_version = 'Unknown' basebackup_version = 'Unknown' if self.server.streaming is not None: pg_version = self.server.streaming.server_txt_version if remote_status['pg_basebackup_version'] is not None: basebackup_version = remote_status['pg_basebackup_version'] hint = "PostgreSQL version: %s, pg_basebackup version: %s" % ( pg_version, basebackup_version ) check_strategy.result( self.config.name, remote_status['pg_basebackup_compatible'], hint=hint) # Skip further checks if the postgres connection doesn't work. # We assume that this error condition will be reported by # another check. postgres = self.server.postgres if postgres is None or postgres.server_txt_version is None: return check_strategy.init_check('pg_basebackup supports tablespaces mapping') # We can't backup a cluster with tablespaces if the tablespace # mapping option is not available in the installed version # of pg_basebackup. pg_version = Version(postgres.server_txt_version) tablespaces_list = postgres.get_tablespaces() # pg_basebackup supports the tablespace-mapping option, # so there are no problems in this case if remote_status['pg_basebackup_tbls_mapping']: hint = None check_result = True # pg_basebackup doesn't support the tablespace-mapping option # and the data directory contains tablespaces, we can't correctly # backup it. elif tablespaces_list: check_result = False if pg_version < '9.3': hint = "pg_basebackup can't be used with tablespaces " \ "and PostgreSQL older than 9.3" else: hint = "pg_basebackup 9.4 or higher is required for " \ "tablespaces support" # Even if pg_basebackup doesn't support the tablespace-mapping # option, this location can be correctly backed up as doesn't # have any tablespaces else: check_result = True if pg_version < '9.3': hint = "pg_basebackup can be used as long as tablespaces " \ "support is not required" else: hint = "pg_basebackup 9.4 or higher is required for " \ "tablespaces support" check_strategy.result( self.config.name, check_result, hint=hint ) def fetch_remote_status(self): """ Gather info from the remote server. This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. """ remote_status = dict.fromkeys( ('pg_basebackup_compatible', 'pg_basebackup_installed', 'pg_basebackup_tbls_mapping', 'pg_basebackup_path', 'pg_basebackup_bwlimit', 'pg_basebackup_version'), None) # Test pg_basebackup existence version_info = PgBaseBackup.get_version_info( self.server.path) if version_info['full_path']: remote_status["pg_basebackup_installed"] = True remote_status["pg_basebackup_path"] = version_info['full_path'] remote_status["pg_basebackup_version"] = ( version_info['full_version']) pgbasebackup_version = version_info['major_version'] else: remote_status["pg_basebackup_installed"] = False return remote_status # Is bandwidth limit supported? if remote_status['pg_basebackup_version'] is not None \ and remote_status['pg_basebackup_version'] < '9.4': remote_status['pg_basebackup_bwlimit'] = False else: remote_status['pg_basebackup_bwlimit'] = True # Is the tablespace mapping option supported? if pgbasebackup_version >= '9.4': remote_status["pg_basebackup_tbls_mapping"] = True else: remote_status["pg_basebackup_tbls_mapping"] = False # Retrieve the PostgreSQL version pg_version = None if self.server.streaming is not None: pg_version = self.server.streaming.server_major_version # If any of the two versions is unknown, we can't compare them if pgbasebackup_version is None or pg_version is None: # Return here. We are unable to retrieve # pg_basebackup or PostgreSQL versions return remote_status # pg_version is not None so transform into a Version object # for easier comparison between versions pg_version = Version(pg_version) # pg_basebackup 9.2 is compatible only with PostgreSQL 9.2. if "9.2" == pg_version == pgbasebackup_version: remote_status["pg_basebackup_compatible"] = True # other versions are compatible with lesser versions of PostgreSQL # WARNING: The development versions of `pg_basebackup` are considered # higher than the stable versions here, but this is not an issue # because it accepts everything that is less than # the `pg_basebackup` version(e.g. '9.6' is less than '9.6devel') elif "9.2" < pg_version <= pgbasebackup_version: remote_status["pg_basebackup_compatible"] = True else: remote_status["pg_basebackup_compatible"] = False return remote_status def backup_copy(self, backup_info): """ Perform the actual copy of the backup using pg_basebackup. First, manages tablespaces, then copies the base backup using the streaming protocol. In case of failure during the execution of the pg_basebackup command the method raises a DataTransferFailure, this trigger the retrying mechanism when necessary. :param barman.infofile.BackupInfo backup_info: backup information """ # Make sure the destination directory exists, ensure the # right permissions to the destination dir backup_dest = backup_info.get_data_directory() dest_dirs = [backup_dest] # Store the start time self.copy_start_time = datetime.datetime.now() # Manage tablespaces, we need to handle them now in order to # be able to relocate them inside the # destination directory of the basebackup tbs_map = {} if backup_info.tablespaces: for tablespace in backup_info.tablespaces: source = tablespace.location destination = backup_info.get_data_directory(tablespace.oid) tbs_map[source] = destination dest_dirs.append(destination) # Prepare the destination directories for pgdata and tablespaces self._prepare_backup_destination(dest_dirs) # Retrieve pg_basebackup version information remote_status = self.get_remote_status() # If pg_basebackup supports --max-rate set the bandwidth_limit bandwidth_limit = None if remote_status['pg_basebackup_bwlimit']: bandwidth_limit = self.config.bandwidth_limit # Make sure we are not wasting precious PostgreSQL resources # for the whole duration of the copy self.server.close() pg_basebackup = PgBaseBackup( connection=self.server.streaming, destination=backup_dest, command=remote_status['pg_basebackup_path'], version=remote_status['pg_basebackup_version'], app_name=self.config.streaming_backup_name, tbs_mapping=tbs_map, bwlimit=bandwidth_limit, immediate=self.config.immediate_checkpoint, path=self.server.path, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, retry_handler=partial(self._retry_handler, dest_dirs)) # Do the actual copy try: pg_basebackup() except CommandFailedException as e: msg = "data transfer failure on directory '%s'" % \ backup_info.get_data_directory() raise DataTransferFailure.from_command_error( 'pg_basebackup', e, msg) # Store the end time self.copy_end_time = datetime.datetime.now() # Store statistics about the copy copy_time = total_seconds(self.copy_end_time - self.copy_start_time) backup_info.copy_stats = { 'copy_time': copy_time, 'total_time': copy_time, } # Check for the presence of configuration files outside the PGDATA external_config = backup_info.get_external_config_files() if any(external_config): msg = ("pg_basebackup does not copy the PostgreSQL " "configuration files that reside outside PGDATA. " "Please manually backup the following files:\n" "\t%s\n" % "\n\t".join(ecf.path for ecf in external_config)) # Show the warning only if the EXTERNAL_CONFIGURATION option # is not specified in the backup_options. if (BackupOptions.EXTERNAL_CONFIGURATION not in self.config.backup_options): output.warning(msg) else: _logger.debug(msg) def _retry_handler(self, dest_dirs, command, args, kwargs, attempt, exc): """ Handler invoked during a backup in case of retry. The method simply warn the user of the failure and remove the already existing directories of the backup. :param list[str] dest_dirs: destination directories :param RsyncPgData command: Command object being executed :param list args: command args :param dict kwargs: command kwargs :param int attempt: attempt number (starting from 0) :param CommandFailedException exc: the exception which caused the failure """ output.warning("Failure executing a backup using pg_basebackup " "(attempt %s)", attempt) output.warning("The files copied so far will be removed and " "the backup process will restart in %s seconds", self.config.basebackup_retry_sleep) # Remove all the destination directories and reinit the backup self._prepare_backup_destination(dest_dirs) def _prepare_backup_destination(self, dest_dirs): """ Prepare the destination of the backup, including tablespaces. This method is also responsible for removing a directory if it already exists and for ensuring the correct permissions for the created directories :param list[str] dest_dirs: destination directories """ for dest_dir in dest_dirs: # Remove a dir if exists. Ignore eventual errors shutil.rmtree(dest_dir, ignore_errors=True) # create the dir mkpath(dest_dir) # Ensure the right permissions to the destination directory # chmod 0700 octal os.chmod(dest_dir, 448) def _start_backup_copy_message(self, backup_info): output.info("Starting backup copy via pg_basebackup for %s", backup_info.backup_id) class SshBackupExecutor(with_metaclass(ABCMeta, BackupExecutor)): """ Abstract base class for any backup executors based on Ssh remote connections. This class is also a factory for exclusive/concurrent backup strategy objects. Raises a SshCommandException if 'ssh_command' is not set. """ def __init__(self, backup_manager, mode): """ Constructor of the abstract class for backups via Ssh :param barman.backup.BackupManager backup_manager: the BackupManager assigned to the executor """ super(SshBackupExecutor, self).__init__(backup_manager, mode) # Retrieve the ssh command and the options necessary for the # remote ssh access. self.ssh_command, self.ssh_options = _parse_ssh_command( backup_manager.config.ssh_command) # Requires ssh_command to be set if not self.ssh_command: raise SshCommandException( 'Missing or invalid ssh_command in barman configuration ' 'for server %s' % backup_manager.config.name) # Apply the default backup strategy if (BackupOptions.CONCURRENT_BACKUP not in self.config.backup_options and BackupOptions.EXCLUSIVE_BACKUP not in self.config.backup_options): self.config.backup_options.add(BackupOptions.EXCLUSIVE_BACKUP) output.debug("The default backup strategy for " "any ssh based backup_method is: " "exclusive_backup") # Depending on the backup options value, create the proper strategy if BackupOptions.CONCURRENT_BACKUP in self.config.backup_options: # Concurrent backup strategy self.strategy = ConcurrentBackupStrategy(self) else: # Exclusive backup strategy self.strategy = ExclusiveBackupStrategy(self) def _update_action_from_strategy(self): """ Update the executor's current action with the one of the strategy. This is used during exception handling to let the caller know where the failure occurred. """ action = getattr(self.strategy, 'current_action', None) if action: self.current_action = action @abstractmethod def backup_copy(self, backup_info): """ Performs the actual copy of a backup for the server :param barman.infofile.BackupInfo backup_info: backup information """ def backup(self, backup_info): """ Perform a backup for the server - invoked by BackupManager.backup() through the generic interface of a BackupExecutor. This implementation is responsible for performing a backup through a remote connection to the PostgreSQL server via Ssh. The specific set of instructions depends on both the specific class that derives from SshBackupExecutor and the selected strategy (e.g. exclusive backup through Rsync). :param barman.infofile.BackupInfo backup_info: backup information """ # Start the backup, all the subsequent code must be wrapped in a # try except block which finally issues a stop_backup command try: self.strategy.start_backup(backup_info) except BaseException: self._update_action_from_strategy() raise try: # save any metadata changed by start_backup() call # This must be inside the try-except, because it could fail backup_info.save() if backup_info.begin_wal is not None: output.info("Backup start at LSN: %s (%s, %08X)", backup_info.begin_xlog, backup_info.begin_wal, backup_info.begin_offset) else: output.info("Backup start at LSN: %s", backup_info.begin_xlog) # If this is the first backup, purge eventually unused WAL files self._purge_unused_wal_files(backup_info) # Start the copy self.current_action = "copying files" self._start_backup_copy_message(backup_info) self.backup_copy(backup_info) self._stop_backup_copy_message(backup_info) # Try again to purge eventually unused WAL files. At this point # the begin_wal value is surely known. Doing it twice is safe # because this function is useful only during the first backup. self._purge_unused_wal_files(backup_info) except: # we do not need to do anything here besides re-raising the # exception. It will be handled in the external try block. output.error("The backup has failed %s", self.current_action) raise else: self.current_action = "issuing stop of the backup" finally: output.info("Asking PostgreSQL server to finalize the backup.") try: self.strategy.stop_backup(backup_info) except BaseException: self._update_action_from_strategy() raise def check(self, check_strategy): """ Perform additional checks for SshBackupExecutor, including Ssh connection (executing a 'true' command on the remote server) and specific checks for the given backup strategy. :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('ssh') hint = "PostgreSQL server" cmd = None minimal_ssh_output = None try: cmd = UnixRemoteCommand(self.ssh_command, self.ssh_options, path=self.server.path) minimal_ssh_output = ''.join(cmd.get_last_output()) except FsOperationFailed as e: hint = str(e).strip() # Output the result check_strategy.result(self.config.name, cmd is not None, hint=hint) # Check if the communication channel is "clean" if minimal_ssh_output: check_strategy.init_check('ssh output clean') check_strategy.result( self.config.name, False, hint="the configured ssh_command must not add anything to " "the remote command output") # If SSH works but PostgreSQL is not responding if (cmd is not None and self.server.get_remote_status().get('server_txt_version') is None): # Check for 'backup_label' presence last_backup = self.server.get_backup( self.server.get_last_backup_id(BackupInfo.STATUS_NOT_EMPTY) ) # Look for the latest backup in the catalogue if last_backup: check_strategy.init_check('backup_label') # Get PGDATA and build path to 'backup_label' backup_label = os.path.join(last_backup.pgdata, 'backup_label') # Verify that backup_label exists in the remote PGDATA. # If so, send an alert. Do not show anything if OK. exists = cmd.exists(backup_label) if exists: hint = "Check that the PostgreSQL server is up " \ "and no 'backup_label' file is in PGDATA." check_strategy.result(self.config.name, False, hint=hint) try: # Invoke specific checks for the backup strategy self.strategy.check(check_strategy) except BaseException: self._update_action_from_strategy() raise def status(self): """ Set additional status info for SshBackupExecutor using remote commands via Ssh, as well as those defined by the given backup strategy. """ try: # Invoke the status() method for the given strategy self.strategy.status() except BaseException: self._update_action_from_strategy() raise def fetch_remote_status(self): """ Get remote information on PostgreSQL using Ssh, such as last archived WAL file This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ remote_status = {} # Retrieve the last archived WAL using a Ssh connection on # the remote server and executing an 'ls' command. Only # for pre-9.4 versions of PostgreSQL. try: if self.server.postgres and \ self.server.postgres.server_version < 90400: remote_status['last_archived_wal'] = None if self.server.postgres.get_setting('data_directory') and \ self.server.postgres.get_setting('archive_command'): cmd = UnixRemoteCommand(self.ssh_command, self.ssh_options, path=self.server.path) # Here the name of the PostgreSQL WALs directory is # hardcoded, but that doesn't represent a problem as # this code runs only for PostgreSQL < 9.4 archive_dir = os.path.join( self.server.postgres.get_setting('data_directory'), 'pg_xlog', 'archive_status') out = str(cmd.list_dir_content(archive_dir, ['-t'])) for line in out.splitlines(): if line.endswith('.done'): name = line[:-5] if xlog.is_any_xlog_file(name): remote_status['last_archived_wal'] = name break except (PostgresConnectionError, FsOperationFailed) as e: _logger.warn("Error retrieving PostgreSQL status: %s", e) return remote_status def _start_backup_copy_message(self, backup_info): number_of_workers = self.config.parallel_jobs message = "Starting backup copy via rsync/SSH for %s" % ( backup_info.backup_id,) if number_of_workers > 1: message += " (%s jobs)" % number_of_workers output.info(message) class RsyncBackupExecutor(SshBackupExecutor): """ Concrete class for backup via Rsync+Ssh. It invokes PostgreSQL commands to start and stop the backup, depending on the defined strategy. Data files are copied using Rsync via Ssh. It heavily relies on methods defined in the SshBackupExecutor class from which it derives. """ PGDATA_EXCLUDE_LIST = [ # Exclude this to avoid log files copy '/pg_log/*', # Exclude this for (PostgreSQL < 10) to avoid WAL files copy '/pg_xlog/*', # This have been renamed on PostgreSQL 10 '/pg_wal/*', # We handle this on a different step of the copy '/global/pg_control', ] EXCLUDE_LIST = [ # Files: see excludeFiles const in PostgreSQL source 'pgsql_tmp*', 'postgresql.auto.conf.tmp', 'postmaster.pid', 'postmaster.opts', 'recovery.conf', # Directories: see excludeDirContents const in PostgreSQL source 'pg_dynshmem/*', 'pg_notify/*', 'pg_replslot/*', 'pg_serial/*', 'pg_stat_tmp/*', 'pg_snapshots/*', 'pg_subtrans/*', ] def __init__(self, backup_manager): """ Constructor :param barman.backup.BackupManager backup_manager: the BackupManager assigned to the strategy """ super(RsyncBackupExecutor, self).__init__(backup_manager, 'rsync') def backup_copy(self, backup_info): """ Perform the actual copy of the backup using Rsync. First, it copies one tablespace at a time, then the PGDATA directory, and finally configuration files (if outside PGDATA). Bandwidth limitation, according to configuration, is applied in the process. This method is the core of base backup copy using Rsync+Ssh. :param barman.infofile.BackupInfo backup_info: backup information """ # Retrieve the previous backup metadata, then calculate safe_horizon previous_backup = self.backup_manager.get_previous_backup( backup_info.backup_id) safe_horizon = None reuse_backup = None # Store the start time self.copy_start_time = datetime.datetime.now() if previous_backup: # safe_horizon is a tz-aware timestamp because BackupInfo class # ensures that property reuse_backup = self.config.reuse_backup safe_horizon = previous_backup.begin_time # Create the copy controller object, specific for rsync, # which will drive all the copy operations. Items to be # copied are added before executing the copy() method controller = RsyncCopyController( path=self.server.path, ssh_command=self.ssh_command, ssh_options=self.ssh_options, network_compression=self.config.network_compression, reuse_backup=reuse_backup, safe_horizon=safe_horizon, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, workers=self.config.parallel_jobs, ) # List of paths to be excluded by the PGDATA copy exclude_and_protect = [] # Process every tablespace if backup_info.tablespaces: for tablespace in backup_info.tablespaces: # If the tablespace location is inside the data directory, # exclude and protect it from being copied twice during # the data directory copy if tablespace.location.startswith(backup_info.pgdata): exclude_and_protect += [ tablespace.location[len(backup_info.pgdata):]] # Exclude and protect the tablespace from being copied again # during the data directory copy exclude_and_protect += ["pg_tblspc/%s" % tablespace.oid] # Make sure the destination directory exists in order for # smart copy to detect that no file is present there tablespace_dest = backup_info.get_data_directory( tablespace.oid) mkpath(tablespace_dest) # Add the tablespace directory to the list of objects # to be copied by the controller. # NOTE: Barman should archive only the content of directory # "PG_" + PG_MAJORVERSION + "_" + CATALOG_VERSION_NO # but CATALOG_VERSION_NO is not easy to retrieve, so we copy # "PG_" + PG_MAJORVERSION + "_*" # It could select some spurious directory if a development or # a beta version have been used, but it's good enough for a # production system as it filters out other major versions. controller.add_directory( label=tablespace.name, src=':%s/' % tablespace.location, dst=tablespace_dest, exclude=['/*'] + self.EXCLUDE_LIST, include=['/PG_%s_*' % self.server.postgres.server_major_version], bwlimit=self.config.get_bwlimit(tablespace), reuse=self._reuse_path(previous_backup, tablespace), item_class=controller.TABLESPACE_CLASS, ) # Make sure the destination directory exists in order for smart copy # to detect that no file is present there backup_dest = backup_info.get_data_directory() mkpath(backup_dest) # Add the PGDATA directory to the list of objects to be copied # by the controller controller.add_directory( label='pgdata', src=':%s/' % backup_info.pgdata, dst=backup_dest, exclude=self.PGDATA_EXCLUDE_LIST + self.EXCLUDE_LIST, exclude_and_protect=exclude_and_protect, bwlimit=self.config.get_bwlimit(), reuse=self._reuse_path(previous_backup), item_class=controller.PGDATA_CLASS, ) # At last copy pg_control controller.add_file( label='pg_control', src=':%s/global/pg_control' % backup_info.pgdata, dst='%s/global/pg_control' % (backup_dest,), item_class=controller.PGCONTROL_CLASS, ) # Copy configuration files (if not inside PGDATA) external_config_files = backup_info.get_external_config_files() included_config_files = [] for config_file in external_config_files: # Add included files to a list, they will be handled later if config_file.file_type == 'include': included_config_files.append(config_file) continue # If the ident file is missing, it isn't an error condition # for PostgreSQL. # Barman is consistent with this behavior. optional = False if config_file.file_type == 'ident_file': optional = True # Create the actual copy jobs in the controller controller.add_file( label=config_file.file_type, src=':%s' % config_file.path, dst=backup_dest, optional=optional, item_class=controller.CONFIG_CLASS, ) # Execute the copy try: controller.copy() # TODO: Improve the exception output except CommandFailedException as e: msg = "data transfer failure" raise DataTransferFailure.from_command_error( 'rsync', e, msg) # Store the end time self.copy_end_time = datetime.datetime.now() # Store statistics about the copy backup_info.copy_stats = controller.statistics() # Check for any include directives in PostgreSQL configuration # Currently, include directives are not supported for files that # reside outside PGDATA. These files must be manually backed up. # Barman will emit a warning and list those files if any(included_config_files): msg = ("The usage of include directives is not supported " "for files that reside outside PGDATA.\n" "Please manually backup the following files:\n" "\t%s\n" % "\n\t".join(icf.path for icf in included_config_files)) # Show the warning only if the EXTERNAL_CONFIGURATION option # is not specified in the backup_options. if (BackupOptions.EXTERNAL_CONFIGURATION not in self.config.backup_options): output.warning(msg) else: _logger.debug(msg) def _reuse_path(self, previous_backup_info, tablespace=None): """ If reuse_backup is 'copy' or 'link', builds the path of the directory to reuse, otherwise always returns None. If oid is None, it returns the full path of PGDATA directory of the previous_backup otherwise it returns the path to the specified tablespace using it's oid. :param barman.infofile.BackupInfo previous_backup_info: backup to be reused :param barman.infofile.Tablespace tablespace: the tablespace to copy :returns: a string containing the local path with data to be reused or None :rtype: str|None """ oid = None if tablespace: oid = tablespace.oid if self.config.reuse_backup in ('copy', 'link') and \ previous_backup_info is not None: try: return previous_backup_info.get_data_directory(oid) except ValueError: return None class BackupStrategy(with_metaclass(ABCMeta, object)): """ Abstract base class for a strategy to be used by a backup executor. """ #: Regex for START WAL LOCATION info START_TIME_RE = re.compile('^START TIME: (.*)', re.MULTILINE) #: Regex for START TIME info WAL_RE = re.compile('^START WAL LOCATION: (.*) \(file (.*)\)', re.MULTILINE) def __init__(self, executor, mode=None): """ Constructor :param BackupExecutor executor: the BackupExecutor assigned to the strategy """ self.executor = executor # Holds the action being executed. Used for error messages. self.current_action = None self.mode = mode def start_backup(self, backup_info): """ Issue a start of a backup - invoked by BackupExecutor.backup() :param barman.infofile.BackupInfo backup_info: backup information """ # Retrieve PostgreSQL server metadata self._pg_get_metadata(backup_info) # Record that we are about to start the backup self.current_action = "issuing start backup command" _logger.debug(self.current_action) @abstractmethod def stop_backup(self, backup_info): """ Issue a stop of a backup - invoked by BackupExecutor.backup() :param barman.infofile.BackupInfo backup_info: backup information """ @abstractmethod def check(self, check_strategy): """ Perform additional checks - invoked by BackupExecutor.check() :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ # noinspection PyMethodMayBeStatic def status(self): """ Set additional status info - invoked by BackupExecutor.status() """ def _pg_get_metadata(self, backup_info): """ Load PostgreSQL metadata into the backup_info parameter :param barman.infofile.BackupInfo backup_info: backup information """ server = self.executor.server # Get the PostgreSQL data directory location self.current_action = 'detecting data directory' output.debug(self.current_action) data_directory = server.postgres.get_setting('data_directory') backup_info.set_attribute('pgdata', data_directory) # Set server version backup_info.set_attribute('version', server.postgres.server_version) # Set XLOG segment size backup_info.set_attribute('xlog_segment_size', server.postgres.xlog_segment_size) # Set configuration files location cf = server.postgres.get_configuration_files() for key in cf: backup_info.set_attribute(key, cf[key]) # Get tablespaces information self.current_action = 'detecting tablespaces' output.debug(self.current_action) tablespaces = server.postgres.get_tablespaces() if tablespaces and len(tablespaces) > 0: backup_info.set_attribute('tablespaces', tablespaces) for item in tablespaces: msg = "\t%s, %s, %s" % (item.oid, item.name, item.location) _logger.info(msg) @staticmethod def _backup_info_from_start_location(backup_info, start_info): """ Fill a backup info with information from a start_backup :param barman.infofile.BackupInfo backup_info: object representing a backup :param DictCursor start_info: the result of the pg_start_backup command """ backup_info.set_attribute('status', "STARTED") backup_info.set_attribute('begin_time', start_info['timestamp']) backup_info.set_attribute('begin_xlog', start_info['location']) # PostgreSQL 9.6+ directly provides the timeline if start_info.get('timeline') is not None: backup_info.set_attribute('timeline', start_info['timeline']) # Take a copy of stop_info because we are going to update it start_info = start_info.copy() start_info.update(xlog.location_to_xlogfile_name_offset( start_info['location'], start_info['timeline'], backup_info.xlog_segment_size)) # If file_name and file_offset are available, use them if (start_info.get('file_name') is not None and start_info.get('file_offset') is not None): backup_info.set_attribute('begin_wal', start_info['file_name']) backup_info.set_attribute('begin_offset', start_info['file_offset']) # If the timeline is still missing, extract it from the file_name if backup_info.timeline is None: backup_info.set_attribute( 'timeline', int(start_info['file_name'][0:8], 16)) @staticmethod def _backup_info_from_stop_location(backup_info, stop_info): """ Fill a backup info with information from a backup stop location :param barman.infofile.BackupInfo backup_info: object representing a backup :param DictCursor stop_info: location info of stop backup """ # If file_name or file_offset are missing build them using the stop # location and the timeline. if (stop_info.get('file_name') is None or stop_info.get('file_offset') is None): # Take a copy of stop_info because we are going to update it stop_info = stop_info.copy() # Get the timeline from the stop_info if available, otherwise # Use the one from the backup_label timeline = stop_info.get('timeline') if timeline is None: timeline = backup_info.timeline stop_info.update(xlog.location_to_xlogfile_name_offset( stop_info['location'], timeline, backup_info.xlog_segment_size)) backup_info.set_attribute('end_time', stop_info['timestamp']) backup_info.set_attribute('end_xlog', stop_info['location']) backup_info.set_attribute('end_wal', stop_info['file_name']) backup_info.set_attribute('end_offset', stop_info['file_offset']) def _backup_info_from_backup_label(self, backup_info): """ Fill a backup info with information from the backup_label file :param barman.infofile.BackupInfo backup_info: object representing a backup """ # If backup_label is present in backup_info use it... if backup_info.backup_label: backup_label_data = backup_info.backup_label # ... otherwise load backup info from backup_label file else: backup_label_path = os.path.join(backup_info.get_data_directory(), 'backup_label') with open(backup_label_path) as backup_label_file: backup_label_data = backup_label_file.read() # Parse backup label wal_info = self.WAL_RE.search(backup_label_data) start_time = self.START_TIME_RE.search(backup_label_data) if wal_info is None or start_time is None: raise ValueError("Failure parsing backup_label for backup %s" % backup_info.backup_id) # Set data in backup_info from backup_label backup_info.set_attribute('timeline', int(wal_info.group(2)[0:8], 16)) backup_info.set_attribute('begin_xlog', wal_info.group(1)) backup_info.set_attribute('begin_wal', wal_info.group(2)) backup_info.set_attribute('begin_offset', xlog.parse_lsn( wal_info.group(1)) % backup_info.xlog_segment_size) backup_info.set_attribute('begin_time', dateutil.parser.parse( start_time.group(1))) class PostgresBackupStrategy(BackupStrategy): """ Concrete class for postgres backup strategy. This strategy is for PostgresBackupExecutor only and is responsible for executing pre e post backup operations during a physical backup executed using pg_basebackup. """ def check(self, check_strategy): """ Perform additional checks for the Postgres backup strategy """ def start_backup(self, backup_info): """ Manage the start of an pg_basebackup backup The method performs all the preliminary operations required for a backup executed using pg_basebackup to start, gathering information from postgres and filling the backup_info. :param barman.infofile.BackupInfo backup_info: backup information """ self.current_action = "initialising postgres backup_method" super(PostgresBackupStrategy, self).start_backup(backup_info) postgres = self.executor.server.postgres current_xlog_info = postgres.current_xlog_info self._backup_info_from_start_location(backup_info, current_xlog_info) def stop_backup(self, backup_info): """ Manage the stop of an pg_basebackup backup The method retrieves the information necessary for the backup.info file reading the backup_label file. Due of the nature of the pg_basebackup, information that are gathered during the start of a backup performed using rsync, are retrieved here :param barman.infofile.BackupInfo backup_info: backup information """ self._backup_info_from_backup_label(backup_info) # Set data in backup_info from current_xlog_info self.current_action = "stopping postgres backup_method" output.info("Finalising the backup.") # Get the current xlog position postgres = self.executor.server.postgres current_xlog_info = postgres.current_xlog_info if current_xlog_info: self._backup_info_from_stop_location( backup_info, current_xlog_info) # Ask PostgreSQL to switch to another WAL file. This is needed # to archive the transaction log file containing the backup # end position, which is required to recover from the backup. try: postgres.switch_wal() except PostgresIsInRecovery: # Skip switching XLOG if a standby server pass class ExclusiveBackupStrategy(BackupStrategy): """ Concrete class for exclusive backup strategy. This strategy is for SshBackupExecutor only and is responsible for coordinating Barman with PostgreSQL on standard physical backup operations (known as 'exclusive' backup), such as invoking pg_start_backup() and pg_stop_backup() on the master server. """ def __init__(self, executor): """ Constructor :param BackupExecutor executor: the BackupExecutor assigned to the strategy """ super(ExclusiveBackupStrategy, self).__init__(executor, 'exclusive') # Make sure that executor is of type SshBackupExecutor assert isinstance(executor, SshBackupExecutor) # Make sure that backup_options does not contain 'concurrent' assert (BackupOptions.CONCURRENT_BACKUP not in self.executor.config.backup_options) def start_backup(self, backup_info): """ Manage the start of an exclusive backup The method performs all the preliminary operations required for an exclusive physical backup to start, as well as preparing the information on the backup for Barman. :param barman.infofile.BackupInfo backup_info: backup information """ super(ExclusiveBackupStrategy, self).start_backup(backup_info) label = "Barman backup %s %s" % ( backup_info.server_name, backup_info.backup_id) # Issue an exclusive start backup command _logger.debug("Start of exclusive backup") postgres = self.executor.server.postgres start_info = postgres.start_exclusive_backup(label) self._backup_info_from_start_location(backup_info, start_info) def stop_backup(self, backup_info): """ Manage the stop of an exclusive backup The method informs the PostgreSQL server that the physical exclusive backup is finished, as well as preparing the information returned by PostgreSQL for Barman. :param barman.infofile.BackupInfo backup_info: backup information """ self.current_action = "issuing stop backup command" _logger.debug("Stop of exclusive backup") stop_info = self.executor.server.postgres.stop_exclusive_backup() self._backup_info_from_stop_location(backup_info, stop_info) def check(self, check_strategy): """ Perform additional checks for ExclusiveBackupStrategy :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ # Make sure PostgreSQL is not in recovery (i.e. is a master) check_strategy.init_check('not in recovery') if self.executor.server.postgres: is_in_recovery = self.executor.server.postgres.is_in_recovery if not is_in_recovery: check_strategy.result( self.executor.config.name, True) else: check_strategy.result( self.executor.config.name, False, hint='cannot perform exclusive backup on a standby') class ConcurrentBackupStrategy(BackupStrategy): """ Concrete class for concurrent backup strategy. This strategy is for SshBackupExecutor only and is responsible for coordinating Barman with PostgreSQL on concurrent physical backup operations through the pgespresso extension. """ def __init__(self, executor): """ Constructor :param BackupExecutor executor: the BackupExecutor assigned to the strategy """ super(ConcurrentBackupStrategy, self).__init__(executor, 'concurrent') # Make sure that executor is of type SshBackupExecutor assert isinstance(executor, SshBackupExecutor) # Make sure that backup_options contains 'concurrent' assert (BackupOptions.CONCURRENT_BACKUP in self.executor.config.backup_options) # noinspection PyMethodMayBeStatic def _write_backup_label(self, backup_info): """ Write backup_label file inside PGDATA folder :param barman.infofile.BackupInfo backup_info: tbackup information """ label_file = os.path.join(backup_info.get_data_directory(), 'backup_label') output.debug("Writing backup label: %s" % label_file) with open(label_file, 'w') as f: f.write(backup_info.backup_label) def _write_tablespace_map(self, backup_info): """ Write tablespace_map file inside PGDATA folder :param barman.infofile.BackupInfo backup_info: backup information """ map_file = os.path.join(backup_info.get_data_directory(), 'tablespace_map') output.debug("Writing tablespace map") with open(map_file, 'w') as f: for tbs in backup_info.tablespaces: # In some cases (i.e. PostgreSQL on windows) a tablespace # can contain a newline or a line feed. PostgreSQL # pg_basebackup code does the same. quoted_location = re.sub(r'([\n\r])', r'\\\1', tbs.location) f.write('%s %s\n' % (tbs.oid, quoted_location)) def start_backup(self, backup_info): """ Start of the backup. The method performs all the preliminary operations required for a backup to start. :param barman.infofile.BackupInfo backup_info: backup information """ super(ConcurrentBackupStrategy, self).start_backup(backup_info) label = "Barman backup %s %s" % ( backup_info.server_name, backup_info.backup_id) pg_version = self.executor.server.postgres.server_version if pg_version >= 90600: # On 9.6+ execute native concurrent start backup _logger.debug("Start of native concurrent backup") self._concurrent_start_backup(backup_info, label) else: # On older Postgres use pgespresso _logger.debug("Start of concurrent backup with pgespresso") self._pgespresso_start_backup(backup_info, label) def stop_backup(self, backup_info): """ Stop backup wrapper :param barman.infofile.BackupInfo backup_info: backup information """ pg_version = self.executor.server.postgres.server_version if pg_version >= 90600: # On 9.6+ execute native concurrent stop backup _logger.debug("Stop of native concurrent backup") self._concurrent_stop_backup(backup_info) else: # On older Postgres use pgespresso _logger.debug("Stop of concurrent backup with pgespresso") self._pgespresso_stop_backup(backup_info) # Write backup_label retrieved from postgres connection self.current_action = "writing backup label" self._write_backup_label(backup_info) # Ask PostgreSQL to switch to another WAL file. This is needed # to archive the transaction log file containing the backup # end position, which is required to recover from the backup. postgres = self.executor.server.postgres try: postgres.switch_wal() except PostgresIsInRecovery: # Skip switching XLOG if a standby server pass def _pgespresso_start_backup(self, backup_info, label): """ Start a concurrent backup using pgespresso :param barman.infofile.BackupInfo backup_info: backup information """ postgres = self.executor.server.postgres backup_info.set_attribute('status', "STARTED") start_info = postgres.pgespresso_start_backup(label) backup_info.set_attribute('backup_label', start_info['backup_label']) self._backup_info_from_backup_label(backup_info) def _pgespresso_stop_backup(self, backup_info): """ Stop a concurrent backup using pgespresso :param barman.infofile.BackupInfo backup_info: backup information """ postgres = self.executor.server.postgres stop_info = postgres.pgespresso_stop_backup(backup_info.backup_label) # Obtain a modifiable copy of stop_info object stop_info = stop_info.copy() # We don't know the exact backup stop location, # so we include the whole segment. stop_info['location'] = xlog.location_from_xlogfile_name_offset( stop_info['end_wal'], 0xFFFFFF) self._backup_info_from_stop_location(backup_info, stop_info) def check(self, check_strategy): """ Perform additional checks for ConcurrentBackupStrategy :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('pgespresso extension') postgres = self.executor.server.postgres try: # We execute this check only if the postgres connection is non None # and the server version is lower than 9.6. On latest PostgreSQL # there is a native API for concurrent backups. if postgres and postgres.server_version < 90600: if postgres.has_pgespresso: check_strategy.result(self.executor.config.name, True) else: check_strategy.result(self.executor.config.name, False, hint='required for concurrent ' 'backups on PostgreSQL %s' % postgres.server_major_version) except PostgresConnectionError: # Skip the check if the postgres connection doesn't work. # We assume that this error condition will be reported by # another check. pass def _concurrent_start_backup(self, backup_info, label): """ Start a concurrent backup using the PostgreSQL 9.6 concurrent backup api :param barman.infofile.BackupInfo backup_info: backup information :param str label: the backup label """ postgres = self.executor.server.postgres start_info = postgres.start_concurrent_backup(label) postgres.allow_reconnect = False self._backup_info_from_start_location(backup_info, start_info) def _concurrent_stop_backup(self, backup_info): """ Stop a concurrent backup using the PostgreSQL 9.6 concurrent backup api :param barman.infofile.BackupInfo backup_info: backup information """ postgres = self.executor.server.postgres stop_info = postgres.stop_concurrent_backup() postgres.allow_reconnect = True backup_info.set_attribute('backup_label', stop_info['backup_label']) self._backup_info_from_stop_location(backup_info, stop_info) barman-2.3/barman/cli.py0000644000076500000240000011551313152223257015744 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module implements the interface with the command line and the logger. """ import logging import os import sys from argparse import SUPPRESS, ArgumentTypeError from contextlib import closing from functools import wraps from argh import ArghParser, arg, expects_obj, named import barman.config import barman.diagnose from barman import output from barman.config import RecoveryOptions from barman.exceptions import BadXlogSegmentName from barman.infofile import BackupInfo from barman.server import Server from barman.utils import configure_logging, drop_privileges, parse_log_level _logger = logging.getLogger(__name__) def check_non_negative(value): """ Check for a positive integer option :param value: str containing the value to check """ if value is None: return None try: int_value = int(value) except Exception: raise ArgumentTypeError("'%s' is not a valid non negative integer" % value) if int_value < 0: raise ArgumentTypeError("'%s' is not a valid non negative integer" % value) return int_value def check_positive(value): """ Check for a positive integer option :param value: str containing the value to check """ if value is None: return None try: int_value = int(value) except Exception: raise ArgumentTypeError("'%s' is not a valid positive integer" % value) if int_value < 1: raise ArgumentTypeError("'%s' is not a valid positive integer" % value) return int_value @named('list-server') @arg('--minimal', help='machine readable output') def list_server(minimal=False): """ List available servers, with useful information """ # Get every server, both inactive and temporarily disabled servers = get_server_list() for name in sorted(servers): server = servers[name] # Exception: manage_server_command is not invoked here # Normally you would call manage_server_command to check if the # server is None and to report inactive and disabled servers, but here # we want all servers and the server cannot be None output.init('list_server', name, minimal=minimal) description = server.config.description # If the server has been manually disabled if not server.config.active: description += " (inactive)" # If server has configuration errors elif server.config.disabled: description += " (WARNING: disabled)" output.result('list_server', name, description) output.close_and_exit() def cron(): """ Run maintenance tasks (global command) """ # Skip inactive and temporarily disabled servers servers = get_server_list(skip_inactive=True, skip_disabled=True) for name in sorted(servers): server = servers[name] # Exception: manage_server_command is not invoked here # Normally you would call manage_server_command to check if the # server is None and to report inactive and disabled servers, # but here we have only active and well configured servers. server.cron() output.close_and_exit() # noinspection PyUnusedLocal def server_completer(prefix, parsed_args, **kwargs): global_config(parsed_args) for conf in barman.__config__.servers(): if conf.name.startswith(prefix): yield conf.name # noinspection PyUnusedLocal def server_completer_all(prefix, parsed_args, **kwargs): global_config(parsed_args) current_list = getattr(parsed_args, 'server_name', None) or () for conf in barman.__config__.servers(): if conf.name.startswith(prefix) and conf.name not in current_list: yield conf.name if len(current_list) == 0 and 'all'.startswith(prefix): yield 'all' # noinspection PyUnusedLocal def backup_completer(prefix, parsed_args, **kwargs): global_config(parsed_args) server = get_server(parsed_args) backups = server.get_available_backups() for backup_id in sorted(backups, reverse=True): if backup_id.startswith(prefix): yield backup_id for special_id in ('latest', 'last', 'oldest', 'first'): if len(backups) > 0 and special_id.startswith(prefix): yield special_id @arg('server_name', nargs='+', completer=server_completer_all, help="specifies the server names for the backup command " "('all' will show all available servers)") @arg('--immediate-checkpoint', help='forces the initial checkpoint to be done as quickly as possible', dest='immediate_checkpoint', action='store_true', default=SUPPRESS) @arg('--no-immediate-checkpoint', help='forces the initial checkpoint to be spreaded', dest='immediate_checkpoint', action='store_false', default=SUPPRESS) @arg('--reuse-backup', nargs='?', choices=barman.config.REUSE_BACKUP_VALUES, default=None, const='link', help='use the previous backup to improve transfer-rate. ' 'If no argument is given "link" is assumed') @arg('--retry-times', help='Number of retries after an error if base backup copy fails.', type=check_non_negative) @arg('--retry-sleep', help='Wait time after a failed base backup copy, before retrying.', type=check_non_negative) @arg('--no-retry', help='Disable base backup copy retry logic.', dest='retry_times', action='store_const', const=0) @arg('--jobs', '-j', help='Run the copy in parallel using NJOBS processes.', type=check_positive, metavar='NJOBS') @expects_obj def backup(args): """ Perform a full backup for the given server (supports 'all') """ servers = get_server_list(args, skip_inactive=True) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command(server, name): continue if args.reuse_backup is not None: server.config.reuse_backup = args.reuse_backup if args.retry_sleep is not None: server.config.basebackup_retry_sleep = args.retry_sleep if args.retry_times is not None: server.config.basebackup_retry_times = args.retry_times if hasattr(args, 'immediate_checkpoint'): server.config.immediate_checkpoint = args.immediate_checkpoint if args.jobs is not None: server.config.parallel_jobs = args.jobs with closing(server): server.backup() output.close_and_exit() @named('list-backup') @arg('server_name', nargs='+', completer=server_completer_all, help="specifies the server name for the command " "('all' will show all available servers)") @arg('--minimal', help='machine readable output', action='store_true') @expects_obj def list_backup(args): """ List available backups for the given server (supports 'all') """ servers = get_server_list(args, skip_inactive=True) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command(server, name): continue output.init('list_backup', name, minimal=args.minimal) with closing(server): server.list_backups() output.close_and_exit() @arg('server_name', nargs='+', completer=server_completer_all, help='specifies the server name for the command') @expects_obj def status(args): """ Shows live information and status of the PostgreSQL server """ servers = get_server_list(args, skip_inactive=True) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command(server, name): continue output.init('status', name) with closing(server): server.status() output.close_and_exit() @named('replication-status') @arg('server_name', nargs='+', completer=server_completer_all, help='specifies the server name for the command') @arg('--minimal', help='machine readable output', action='store_true') @arg('--target', choices=('all', 'hot-standby', 'wal-streamer'), default='all', help=''' Possible values are: 'hot-standby' (only hot standby servers), 'wal-streamer' (only WAL streaming clients, such as pg_receivexlog), 'all' (any of them). Defaults to %(default)s''') @expects_obj def replication_status(args): """ Shows live information and status of any streaming client """ servers = get_server_list(args, skip_inactive=True) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command(server, name): continue with closing(server): output.init('replication_status', name, minimal=args.minimal) server.replication_status(args.target) output.close_and_exit() @arg('server_name', nargs='+', completer=server_completer_all, help='specifies the server name for the command') @expects_obj def rebuild_xlogdb(args): """ Rebuild the WAL file database guessing it from the disk content. """ servers = get_server_list(args, skip_inactive=True) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command(server, name): continue with closing(server): server.rebuild_xlogdb() output.close_and_exit() @arg('server_name', completer=server_completer, help='specifies the server name for the command') @arg('--target-tli', help='target timeline', type=check_positive) @arg('--target-time', help='target time. You can use any valid unambiguous representation. ' 'e.g: "YYYY-MM-DD HH:MM:SS.mmm"') @arg('--target-xid', help='target transaction ID') @arg('--target-name', help='target name created previously with ' 'pg_create_restore_point() function call') @arg('--target-immediate', help='end recovery as soon as a consistent state is reached', action='store_true', default=False) @arg('--exclusive', help='set target xid to be non inclusive', action="store_true") @arg('--tablespace', help='tablespace relocation rule', metavar='NAME:LOCATION', action='append') @arg('--remote-ssh-command', metavar='SSH_COMMAND', help='This options activates remote recovery, by specifying the secure ' 'shell command to be launched on a remote host. It is ' 'the equivalent of the "ssh_command" server option in ' 'the configuration file for remote recovery. ' 'Example: "ssh postgres@db2"') @arg('backup_id', completer=backup_completer, help='specifies the backup ID to recover') @arg('destination_directory', help='the directory where the new server is created') @arg('--retry-times', help='Number of retries after an error if base backup copy fails.', type=check_non_negative) @arg('--retry-sleep', help='Wait time after a failed base backup copy, before retrying.', type=check_non_negative) @arg('--no-retry', help='Disable base backup copy retry logic.', dest='retry_times', action='store_const', const=0) @arg('--jobs', '-j', help='Run the copy in parallel using NJOBS processes.', type=check_positive, metavar='NJOBS') @arg('--get-wal', help='Enable the get-wal option during the recovery.', dest='get_wal', action='store_true', default=SUPPRESS) @arg('--no-get-wal', help='Disable the get-wal option during recovery.', dest='get_wal', action='store_false', default=SUPPRESS) @arg('--network-compression', help='Enable network compression during remote recovery.', dest='network_compression', action='store_true', default=SUPPRESS) @arg('--no-network-compression', help='Disable network compression during remote recovery.', dest='network_compression', action='store_false', default=SUPPRESS) @expects_obj def recover(args): """ Recover a server at a given time or xid """ server = get_server(args) # Retrieves the backup backup_id = parse_backup_id(server, args) if backup_id.status != BackupInfo.DONE: output.error( "Cannot recover from backup '%s' of server '%s': " "backup status is not DONE", args.backup_id, server.config.name) output.close_and_exit() # decode the tablespace relocation rules tablespaces = {} if args.tablespace: for rule in args.tablespace: try: tablespaces.update([rule.split(':', 1)]) except ValueError: output.error( "Invalid tablespace relocation rule '%s'\n" "HINT: The valid syntax for a relocation rule is " "NAME:LOCATION", rule) output.close_and_exit() # validate the rules against the tablespace list valid_tablespaces = [] if backup_id.tablespaces: valid_tablespaces = [tablespace_data.name for tablespace_data in backup_id.tablespaces] for item in tablespaces: if item not in valid_tablespaces: output.error("Invalid tablespace name '%s'\n" "HINT: Please use any of the following " "tablespaces: %s", item, ', '.join(valid_tablespaces)) output.close_and_exit() # explicitly disallow the rsync remote syntax (common mistake) if ':' in args.destination_directory: output.error( "The destination directory parameter " "cannot contain the ':' character\n" "HINT: If you want to do a remote recovery you have to use " "the --remote-ssh-command option") output.close_and_exit() if args.retry_sleep is not None: server.config.basebackup_retry_sleep = args.retry_sleep if args.retry_times is not None: server.config.basebackup_retry_times = args.retry_times if hasattr(args, 'get_wal'): if args.get_wal: server.config.recovery_options.add(RecoveryOptions.GET_WAL) else: server.config.recovery_options.remove(RecoveryOptions.GET_WAL) if args.jobs is not None: server.config.parallel_jobs = args.jobs # PostgreSQL supports multiple parameters to specify when the recovery # process will end, and in that case the last entry in recovery.conf # will be used. See [1] # # Since the meaning of the target options is not dependent on the order # of parameters, we decided to make the target options mutually exclusive. # # [1]: https://www.postgresql.org/docs/current/static/ # recovery-target-settings.html target_options = ['target_tli', 'target_time', 'target_xid', 'target_name', 'target_immediate'] specified_target_options = len( [option for option in target_options if getattr(args, option)]) if specified_target_options > 1: output.error( "You cannot specify multiple targets for the recovery operation") output.close_and_exit() if hasattr(args, 'network_compression'): if args.network_compression and args.remote_ssh_command is None: output.error( "Network compression can only be used with " "remote recovery.\n" "HINT: If you want to do a remote recovery " "you have to use the --remote-ssh-command option") output.close_and_exit() server.config.network_compression = args.network_compression with closing(server): server.recover(backup_id, args.destination_directory, tablespaces=tablespaces, target_tli=args.target_tli, target_time=args.target_time, target_xid=args.target_xid, target_name=args.target_name, target_immediate=args.target_immediate, exclusive=args.exclusive, remote_command=args.remote_ssh_command) output.close_and_exit() @named('show-server') @arg('server_name', nargs='+', completer=server_completer_all, help="specifies the server names to show " "('all' will show all available servers)") @expects_obj def show_server(args): """ Show all configuration parameters for the specified servers """ servers = get_server_list(args) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command( server, name, skip_inactive=False, skip_disabled=False, disabled_is_error=False): continue # If the server has been manually disabled if not server.config.active: name += " (inactive)" # If server has configuration errors elif server.config.disabled: name += " (WARNING: disabled)" output.init('show_server', name) with closing(server): server.show() output.close_and_exit() @named('switch-wal') @arg('server_name', nargs='+', completer=server_completer_all, help="specifies the server name target of the switch-wal command") @arg('--force', help='forces the switch of a WAL by executing a checkpoint before', dest='force', action='store_true', default=False) @arg('--archive', help='wait for one WAL file to be archived', dest='archive', action='store_true', default=False) @arg('--archive-timeout', help='the time, in seconds, the archiver will wait for a new WAL file ' 'to be archived before timing out', metavar='TIMEOUT', default='30', type=check_non_negative) @expects_obj def switch_wal(args): """ Execute the switch-wal command on the target server """ servers = get_server_list(args, skip_inactive=True) for name in sorted(servers): server = servers[name] # Skip the server (apply general rule) if not manage_server_command(server, name): continue with closing(server): server.switch_wal(args.force, args.archive, args.archive_timeout) output.close_and_exit() @named('switch-xlog') # Set switch-xlog as alias of switch-wal. # We cannot use the @argh.aliases decorator, because it needs Python >= 3.2, # so we create a wraqpper function and use @wraps to copy all the function # attributes needed by argh @wraps(switch_wal) def switch_xlog(args): return switch_wal(args) @arg('server_name', nargs='+', completer=server_completer_all, help="specifies the server names to check " "('all' will check all available servers)") @arg('--nagios', help='Nagios plugin compatible output', action='store_true') @expects_obj def check(args): """ Check if the server configuration is working. This command returns success if every checks pass, or failure if any of these fails """ if args.nagios: output.set_output_writer(output.NagiosOutputWriter()) servers = get_server_list(args) for name in sorted(servers): server = servers[name] # Validate the returned server if not manage_server_command( server, name, skip_inactive=False, skip_disabled=False, disabled_is_error=False): continue # If the server has been manually disabled if not server.config.active: name += " (inactive)" # If server has configuration errors elif server.config.disabled: name += " (WARNING: disabled)" output.init('check', name, server.config.active) with closing(server): server.check() output.close_and_exit() def diagnose(): """ Diagnostic command (for support and problems detection purpose) """ # Get every server (both inactive and temporarily disabled) servers = get_server_list(on_error_stop=False, suppress_error=True) # errors list with duplicate paths between servers errors_list = barman.__config__.servers_msg_list barman.diagnose.exec_diagnose(servers, errors_list) output.close_and_exit() @named('show-backup') @arg('server_name', completer=server_completer, help='specifies the server name for the command') @arg('backup_id', completer=backup_completer, help='specifies the backup ID') @expects_obj def show_backup(args): """ This method shows a single backup information """ server = get_server(args) # Retrieves the backup backup_info = parse_backup_id(server, args) with closing(server): server.show_backup(backup_info) output.close_and_exit() @named('list-files') @arg('server_name', completer=server_completer, help='specifies the server name for the command') @arg('backup_id', completer=backup_completer, help='specifies the backup ID') @arg('--target', choices=('standalone', 'data', 'wal', 'full'), default='standalone', help=''' Possible values are: data (just the data files), standalone (base backup files, including required WAL files), wal (just WAL files between the beginning of base backup and the following one (if any) or the end of the log) and full (same as data + wal). Defaults to %(default)s''') @expects_obj def list_files(args): """ List all the files for a single backup """ server = get_server(args) # Retrieves the backup backup_info = parse_backup_id(server, args) try: for line in backup_info.get_list_of_files(args.target): output.info(line, log=False) except BadXlogSegmentName as e: output.error( "invalid xlog segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", str(e), server.config.name) output.close_and_exit() @arg('server_name', completer=server_completer, help='specifies the server name for the command') @arg('backup_id', completer=backup_completer, help='specifies the backup ID') @expects_obj def delete(args): """ Delete a backup """ server = get_server(args) # Retrieves the backup backup_id = parse_backup_id(server, args) with closing(server): server.delete_backup(backup_id) output.close_and_exit() @named('get-wal') @arg('server_name', completer=server_completer, help='specifies the server name for the command') @arg('wal_name', help='the WAL file to get') @arg('--output-directory', '-o', help='put the retrieved WAL file in this directory ' 'with the original name', default=SUPPRESS) @arg('--gzip', '-z', '-x', help='compress the output with gzip', action='store_const', const='gzip', dest='compression', default=SUPPRESS) @arg('--bzip2', '-j', help='compress the output with bzip2', action='store_const', const='bzip2', dest='compression', default=SUPPRESS) @arg('--peek', '-p', help="peek from the WAL archive up to 'SIZE' WAL files, starting " "from the requested one. 'SIZE' must be an integer >= 1. " "When invoked with this option, get-wal returns a list of " "zero to 'SIZE' WAL segment names, one per row.", metavar='SIZE', type=check_positive, default=SUPPRESS) @expects_obj def get_wal(args): """ Retrieve WAL_NAME file from SERVER_NAME archive. The content will be streamed on standard output unless the --output-directory option is specified. """ server = get_server(args, inactive_is_error=True) # Retrieve optional arguments. If an argument is not specified, # the namespace doesn't contain it due to SUPPRESS default. # In that case we pick 'None' using getattr third argument. compression = getattr(args, 'compression', None) output_directory = getattr(args, 'output_directory', None) peek = getattr(args, 'peek', None) with closing(server): server.get_wal(args.wal_name, compression=compression, output_directory=output_directory, peek=peek) output.close_and_exit() @named('archive-wal') @arg('server_name', completer=server_completer, help='specifies the server name for the command') @expects_obj def archive_wal(args): """ Execute maintenance operations on WAL files for a given server. This command processes any incoming WAL files for the server and archives them along the catalogue. """ server = get_server(args) with closing(server): server.archive_wal() output.close_and_exit() @named('receive-wal') @arg('--stop', help='stop the receive-wal subprocess for the server', action='store_true') @arg('--reset', help='reset the status of receive-wal removing ' 'any status files', action='store_true') @arg('--create-slot', help='create the replication slot, if it does not exist', action='store_true') @arg('--drop-slot', help='drop the replication slot, if it exists', action='store_true') @arg('server_name', completer=server_completer, help='specifies the server name for the command') @expects_obj def receive_wal(args): """ Start a receive-wal process. The process uses the streaming protocol to receive WAL files from the PostgreSQL server. """ server = get_server(args) if args.stop and args.reset: output.error("--stop and --reset options are not compatible") # If the caller requested to shutdown the receive-wal process deliver the # termination signal, otherwise attempt to start it elif args.stop: server.kill('receive-wal') elif args.create_slot: with closing(server): server.create_physical_repslot() elif args.drop_slot: with closing(server): server.drop_repslot() else: with closing(server): server.receive_wal(reset=args.reset) output.close_and_exit() def pretty_args(args): """ Prettify the given argh namespace to be human readable :type args: argh.dispatching.ArghNamespace :return: the human readable content of the namespace """ values = dict(vars(args)) # Retrieve the command name with recent argh versions if '_functions_stack' in values: values['command'] = values['_functions_stack'][0].__name__ del values['_functions_stack'] # Older argh versions only have the matching function in the namespace elif 'function' in values: values['command'] = values['function'].__name__ del values['function'] return "%r" % values def global_config(args): """ Set the configuration file """ if hasattr(args, 'config'): filename = args.config else: try: filename = os.environ['BARMAN_CONFIG_FILE'] except KeyError: filename = None config = barman.config.Config(filename) barman.__config__ = config # change user if needed try: drop_privileges(config.user) except OSError: msg = "ERROR: please run barman as %r user" % config.user raise SystemExit(msg) except KeyError: msg = "ERROR: the configured user %r does not exists" % config.user raise SystemExit(msg) # configure logging log_level = parse_log_level(config.log_level) configure_logging(config.log_file, log_level or barman.config.DEFAULT_LOG_LEVEL, config.log_format) if log_level is None: _logger.warn('unknown log_level in config file: %s', config.log_level) # configure output if args.format != output.DEFAULT_WRITER or args.quiet or args.debug: output.set_output_writer(args.format, quiet=args.quiet, debug=args.debug) # Load additional configuration files config.load_configuration_files_directory() # We must validate the configuration here in order to have # both output and logging configured config.validate_global_config() _logger.debug('Initialised Barman version %s (config: %s, args: %s)', barman.__version__, config.config_file, pretty_args(args)) def get_server(args, skip_inactive=True, skip_disabled=False, inactive_is_error=False, on_error_stop=True, suppress_error=False): """ Get a single server retrieving its configuration (wraps get_server_list()) Returns a Server object or None if the required server is unknown and on_error_stop is False. WARNING: this function modifies the 'args' parameter :param args: an argparse namespace containing a single server_name parameter WARNING: the function modifies the content of this parameter :param bool skip_inactive: do nothing if the server is inactive :param bool skip_disabled: do nothing if the server is disabled :param bool inactive_is_error: treat inactive server as error :param bool on_error_stop: stop if an error is found :param bool suppress_error: suppress display of errors (e.g. diagnose) :rtype: barman.server.Server|None """ # This function must to be called with in a single-server context name = args.server_name assert isinstance(name, str) # The 'all' special name is forbidden in this context if name == 'all': output.error("You cannot use 'all' in a single server context") output.close_and_exit() # The following return statement will never be reached # but it is here for clarity return None # Builds a list from a single given name args.server_name = [name] # Skip_inactive is reset if inactive_is_error is set, because # it needs to retrieve the inactive server to emit the error. skip_inactive &= not inactive_is_error # Retrieve the requested server servers = get_server_list(args, skip_inactive, skip_disabled, on_error_stop, suppress_error) # The requested server has been excluded from get_server_list result if len(servers) == 0: output.close_and_exit() # The following return statement will never be reached # but it is here for clarity return None # retrieve the server object server = servers[name] # Apply standard validation control and skips # the server if inactive or disabled, displaying standard # error messages. If on_error_stop (default) exits if not manage_server_command(server, name, inactive_is_error) and \ on_error_stop: output.close_and_exit() # The following return statement will never be reached # but it is here for clarity return None # Returns the filtered server return server def get_server_list(args=None, skip_inactive=False, skip_disabled=False, on_error_stop=True, suppress_error=False): """ Get the server list from the configuration If args the parameter is None or arg.server_name is ['all'] returns all defined servers :param args: an argparse namespace containing a list server_name parameter :param bool skip_inactive: skip inactive servers when 'all' is required :param bool skip_disabled: skip disabled servers when 'all' is required :param bool on_error_stop: stop if an error is found :param bool suppress_error: suppress display of errors (e.g. diagnose) :rtype: dict(str,barman.server.Server|None) """ server_dict = {} # This function must to be called with in a multiple-server context assert not args or isinstance(args.server_name, list) # Generate the list of servers (required for global errors) available_servers = barman.__config__.server_names() # Get a list of configuration errors from all the servers global_error_list = barman.__config__.servers_msg_list # Global errors have higher priority if global_error_list: # Output the list of global errors if not suppress_error: for error in global_error_list: output.error(error) # If requested, exit on first error if on_error_stop: output.close_and_exit() # The following return statement will never be reached # but it is here for clarity return {} # Handle special 'all' server cases # - args is None # - 'all' special name if not args or 'all' in args.server_name: # When 'all' is used, it must be the only specified argument if args and len(args.server_name) != 1: output.error("You cannot use 'all' with other server names") servers = available_servers else: servers = args.server_name # Loop through all the requested servers for server in servers: conf = barman.__config__.get_server(server) if conf is None: # Unknown server server_dict[server] = None else: server_object = Server(conf) # Skip inactive servers, if requested if skip_inactive and not server_object.config.active: output.info("Skipping inactive server '%s'" % conf.name) continue # Skip disabled servers, if requested if skip_disabled and server_object.config.disabled: output.info("Skipping temporarily disabled server '%s'" % conf.name) continue server_dict[server] = server_object return server_dict def manage_server_command(server, name=None, inactive_is_error=False, disabled_is_error=True, skip_inactive=True, skip_disabled=True): """ Standard and consistent method for managing server errors within a server command execution. By default, suggests to skip any inactive and disabled server; it also emits errors for disabled servers by default. Returns True if the command has to be executed for this server. :param barman.server.Server server: server to be checked for errors :param str name: name of the server, in a multi-server command :param bool inactive_is_error: treat inactive server as error :param bool disabled_is_error: treat disabled server as error :param bool skip_inactive: skip if inactive :param bool skip_disabled: skip if disabled :return: True if the command has to be executed on this server :rtype: boolean """ # Unknown server (skip it) if not server: output.error("Unknown server '%s'" % name) return False if not server.config.active: # Report inactive server as error if inactive_is_error: output.error('Inactive server: %s' % server.config.name) if skip_inactive: return False # Report disabled server as error if server.config.disabled: # Output all the messages as errors, and exit terminating the run. if disabled_is_error: for message in server.config.msg_list: output.error(message) if skip_disabled: return False # All ok, execute the command return True def parse_backup_id(server, args): """ Parses backup IDs including special words such as latest, oldest, etc. Exit with error if the backup id doesn't exist. :param Server server: server object to search for the required backup :param args: command lien arguments namespace :rtype: BackupInfo """ if args.backup_id in ('latest', 'last'): backup_id = server.get_last_backup_id() elif args.backup_id in ('oldest', 'first'): backup_id = server.get_first_backup_id() else: backup_id = args.backup_id backup_info = server.get_backup(backup_id) if backup_info is None: output.error( "Unknown backup '%s' for server '%s'", args.backup_id, server.config.name) output.close_and_exit() return backup_info def main(): """ The main method of Barman """ p = ArghParser(epilog='Barman by 2ndQuadrant (www.2ndQuadrant.com)') p.add_argument('-v', '--version', action='version', version='%s\n\nBarman by 2ndQuadrant (www.2ndQuadrant.com)' % barman.__version__) p.add_argument('-c', '--config', help='uses a configuration file ' '(defaults: %s)' % ', '.join(barman.config.Config.CONFIG_FILES), default=SUPPRESS) p.add_argument('-q', '--quiet', help='be quiet', action='store_true') p.add_argument('-d', '--debug', help='debug output', action='store_true') p.add_argument('-f', '--format', help='output format', choices=output.AVAILABLE_WRITERS.keys(), default=output.DEFAULT_WRITER) p.add_commands( [ archive_wal, backup, check, cron, delete, diagnose, get_wal, list_backup, list_files, list_server, rebuild_xlogdb, receive_wal, recover, show_backup, show_server, replication_status, status, switch_wal, switch_xlog, ] ) # noinspection PyBroadException try: p.dispatch(pre_call=global_config) except KeyboardInterrupt: msg = "Process interrupted by user (KeyboardInterrupt)" output.error(msg) except Exception as e: msg = "%s\nSee log file for more details." % e output.exception(msg) # cleanup output API and exit honoring output.error_occurred and # output.error_exit_code output.close_and_exit() if __name__ == '__main__': # This code requires the mock module and allow us to test # bash completion inside the IDE debugger try: # noinspection PyUnresolvedReferences import mock sys.stdout = mock.Mock(wraps=sys.stdout) sys.stdout.isatty.return_value = True os.dup2(2, 8) except ImportError: pass main() barman-2.3/barman/command_wrappers.py0000644000076500000240000011654413153276623020551 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module contains a wrapper for shell commands """ from __future__ import print_function import errno import inspect import logging import os import select import signal import subprocess import sys import time from distutils.version import LooseVersion as Version import barman.utils from barman.exceptions import CommandFailedException, CommandMaxRetryExceeded _logger = logging.getLogger(__name__) class StreamLineProcessor(object): """ Class deputed to reading lines from a file object, using a buffered read. NOTE: This class never call os.read() twice in a row. And is designed to work with the select.select() method. """ def __init__(self, fobject, handler): """ :param file fobject: The file that is being read :param callable handler: The function (taking only one unicode string argument) which will be called for every line """ self._file = fobject self._handler = handler self._buf = '' def fileno(self): """ Method used by select.select() to get the underlying file descriptor. :rtype: the underlying file descriptor """ return self._file.fileno() def process(self): """ Read the ready data from the stream and for each line found invoke the handler. :return bool: True when End Of File has been reached """ data = os.read(self._file.fileno(), 4096) # If nothing has been read, we reached the EOF if not data: self._file.close() # Handle the last line (always incomplete, maybe empty) self._handler(self._buf) return True self._buf += data.decode('utf-8') # If no '\n' is present, we just read a part of a very long line. # Nothing to do at the moment. if '\n' not in self._buf: return False tmp = self._buf.split('\n') # Leave the remainder in self._buf self._buf = tmp[-1] # Call the handler for each complete line. lines = tmp[:-1] for line in lines: self._handler(line) return False class Command(object): """ Wrapper for a system command """ def __init__(self, cmd, args=None, env_append=None, path=None, shell=False, check=False, allowed_retval=(0,), close_fds=True, out_handler=None, err_handler=None, retry_times=0, retry_sleep=0, retry_handler=None): """ If the `args` argument is specified the arguments will be always added to the ones eventually passed with the actual invocation. If the `env_append` argument is present its content will be appended to the environment of every invocation. The subprocess output and error stream will be processed through the output and error handler, respectively defined through the `out_handler` and `err_handler` arguments. If not provided every line will be sent to the log respectively at INFO and WARNING level. The `out_handler` and the `err_handler` functions will be invoked with one single argument, which is a string containing the line that is being processed. If the `close_fds` argument is True, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. If the `check` argument is True, the exit code will be checked against the `allowed_retval` list, raising a CommandFailedException if not in the list. If `retry_times` is greater than 0, when the execution of a command terminates with an error, it will be retried for a maximum of `retry_times` times, waiting for `retry_sleep` seconds between every attempt. Everytime a command is retried the `retry_handler` is executed before running the command again. The retry_handler must be a callable that accepts the following fields: * the Command object * the arguments list * the keyword arguments dictionary * the number of the failed attempt * the exception containing the error An example of such a function is: > def retry_handler(command, args, kwargs, attempt, exc): > print("Failed command!") Some of the keyword arguments can be specified both in the class constructor and during the method call. If specified in both places, the method arguments will take the precedence over the constructor arguments. :param str cmd: The command to exexute :param list[str]|None args: List of additional arguments to append :param dict[str.str]|None env_append: additional environment variables :param str path: PATH to be used while searching for `cmd` :param bool shell: If true, use the shell instead of an "execve" call :param bool check: Raise a CommandFailedException if the exit code is not present in `allowed_retval` :param list[int] allowed_retval: List of exit codes considered as a successful termination. :param bool close_fds: If set, close all the extra file descriptors :param callable out_handler: handler for lines sent on stdout :param callable err_handler: handler for lines sent on stderr :param int retry_times: number of allowed retry attempts :param int retry_sleep: wait seconds between every retry :param callable retry_handler: handler invoked during a command retry """ self.pipe = None self.cmd = cmd self.args = args if args is not None else [] self.shell = shell self.close_fds = close_fds self.check = check self.allowed_retval = allowed_retval self.retry_times = retry_times self.retry_sleep = retry_sleep self.retry_handler = retry_handler self.path = path self.ret = None self.out = None self.err = None # If env_append has been provided use it or replace with an empty dict env_append = env_append or {} # If path has been provided, replace it in the environment if path: env_append['PATH'] = path # Find the absolute path to the command to execute if not self.shell: full_path = barman.utils.which(self.cmd, self.path) if not full_path: raise CommandFailedException( '%s not in PATH' % self.cmd) self.cmd = full_path # If env_append contains anything, build an env dict to be used during # subprocess call, otherwise set it to None and let the subprocesses # inherit the parent environment if env_append: self.env = os.environ.copy() self.env.update(env_append) else: self.env = None # If an output handler has been provided use it, otherwise log the # stdout as INFO if out_handler: self.out_handler = out_handler else: self.out_handler = self.make_logging_handler(logging.INFO) # If an error handler has been provided use it, otherwise log the # stderr as WARNING if err_handler: self.err_handler = err_handler else: self.err_handler = self.make_logging_handler(logging.WARNING) @staticmethod def _restore_sigpipe(): """restore default signal handler (http://bugs.python.org/issue1652)""" signal.signal(signal.SIGPIPE, signal.SIG_DFL) # pragma: no cover def __call__(self, *args, **kwargs): """ Run the command and return the exit code. The output and error strings are not returned, but they can be accessed as attributes of the Command object, as well as the exit code. If `stdin` argument is specified, its content will be passed to the executed command through the standard input descriptor. If the `close_fds` argument is True, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. If the `check` argument is True, the exit code will be checked against the `allowed_retval` list, raising a CommandFailedException if not in the list. Every keyword argument can be specified both in the class constructor and during the method call. If specified in both places, the method arguments will take the precedence over the constructor arguments. :rtype: int :raise: CommandFailedException :raise: CommandMaxRetryExceeded """ self.get_output(*args, **kwargs) return self.ret def get_output(self, *args, **kwargs): """ Run the command and return the output and the error as a tuple. The return code is not returned, but it can be accessed as an attribute of the Command object, as well as the output and the error strings. If `stdin` argument is specified, its content will be passed to the executed command through the standard input descriptor. If the `close_fds` argument is True, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. If the `check` argument is True, the exit code will be checked against the `allowed_retval` list, raising a CommandFailedException if not in the list. Every keyword argument can be specified both in the class constructor and during the method call. If specified in both places, the method arguments will take the precedence over the constructor arguments. :rtype: tuple[str, str] :raise: CommandFailedException :raise: CommandMaxRetryExceeded """ attempt = 0 while True: try: return self._get_output_once(*args, **kwargs) except CommandFailedException as exc: # Try again if retry number is lower than the retry limit if attempt < self.retry_times: # If a retry_handler is defined, invoke it passing the # Command instance and the exception if self.retry_handler: self.retry_handler(self, args, kwargs, attempt, exc) # Sleep for configured time, then try again time.sleep(self.retry_sleep) attempt += 1 else: if attempt == 0: # No retry requested by the user # Raise the original exception raise else: # If the max number of attempts is reached and # there is still an error, exit raising # a CommandMaxRetryExceeded exception and wrap the # original one raise CommandMaxRetryExceeded(exc) def _get_output_once(self, *args, **kwargs): """ Run the command and return the output and the error as a tuple. The return code is not returned, but it can be accessed as an attribute of the Command object, as well as the output and the error strings. If `stdin` argument is specified, its content will be passed to the executed command through the standard input descriptor. If the `close_fds` argument is True, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. If the `check` argument is True, the exit code will be checked against the `allowed_retval` list, raising a CommandFailedException if not in the list. Every keyword argument can be specified both in the class constructor and during the method call. If specified in both places, the method arguments will take the precedence over the constructor arguments. :rtype: tuple[str, str] :raises: CommandFailedException """ out = [] err = [] # If check is true, it must be handled here check = kwargs.pop('check', self.check) allowed_retval = kwargs.pop('allowed_retval', self.allowed_retval) self.execute(out_handler=out.append, err_handler=err.append, check=False, *args, **kwargs) self.out = '\n'.join(out) self.err = '\n'.join(err) _logger.debug("Command stdout: %s", self.out) _logger.debug("Command stderr: %s", self.err) # Raise if check and the return code is not in the allowed list if check: self.check_return_value(allowed_retval) return self.out, self.err def check_return_value(self, allowed_retval): """ Check the current return code and raise CommandFailedException when it's not in the allowed_retval list :param list[int] allowed_retval: list of return values considered success :raises: CommandFailedException """ if self.ret not in allowed_retval: raise CommandFailedException(dict( ret=self.ret, out=self.out, err=self.err)) def execute(self, *args, **kwargs): """ Execute the command and pass the output to the configured handlers If `stdin` argument is specified, its content will be passed to the executed command through the standard input descriptor. The subprocess output and error stream will be processed through the output and error handler, respectively defined through the `out_handler` and `err_handler` arguments. If not provided every line will be sent to the log respectively at INFO and WARNING level. If the `close_fds` argument is True, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. If the `check` argument is True, the exit code will be checked against the `allowed_retval` list, raising a CommandFailedException if not in the list. Every keyword argument can be specified both in the class constructor and during the method call. If specified in both places, the method arguments will take the precedence over the constructor arguments. :rtype: int :raise: CommandFailedException """ # Check keyword arguments stdin = kwargs.pop('stdin', None) check = kwargs.pop('check', self.check) allowed_retval = kwargs.pop('allowed_retval', self.allowed_retval) close_fds = kwargs.pop('close_fds', self.close_fds) out_handler = kwargs.pop('out_handler', self.out_handler) err_handler = kwargs.pop('err_handler', self.err_handler) if len(kwargs): raise TypeError('%s() got an unexpected keyword argument %r' % (inspect.stack()[1][3], kwargs.popitem()[0])) # Reset status self.ret = None self.out = None self.err = None # Create the subprocess and save it in the current object to be usable # by signal handlers pipe = self._build_pipe(args, close_fds) self.pipe = pipe # Send the provided input and close the stdin descriptor if stdin: pipe.stdin.write(stdin) pipe.stdin.close() # Prepare the list of processors processors = [ StreamLineProcessor( pipe.stdout, out_handler), StreamLineProcessor( pipe.stderr, err_handler)] # Read the streams until the subprocess exits self.pipe_processor_loop(processors) # Reap the zombie and read the exit code pipe.wait() self.ret = pipe.returncode # Remove the closed pipe from the object self.pipe = None _logger.debug("Command return code: %s", self.ret) # Raise if check and the return code is not in the allowed list if check: self.check_return_value(allowed_retval) return self.ret def _build_pipe(self, args, close_fds): """ Build the Pipe object used by the Command The resulting command will be composed by: self.cmd + self.args + args :param args: extra arguments for the subprocess :param close_fds: if True all file descriptors except 0, 1 and 2 will be closed before the child process is executed. :rtype: subprocess.Popen """ # Append the argument provided to this method ot the base argument list args = self.args + list(args) # If shell is True, properly quote the command if self.shell: cmd = full_command_quote(self.cmd, args) else: cmd = [self.cmd] + args # Log the command we are about to execute _logger.debug("Command: %r", cmd) return subprocess.Popen(cmd, shell=self.shell, env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=self._restore_sigpipe, close_fds=close_fds) @staticmethod def pipe_processor_loop(processors): """ Process the output received through the pipe until all the provided StreamLineProcessor reach the EOF. :param list[StreamLineProcessor] processors: a list of StreamLineProcessor """ # Loop until all the streams reaches the EOF while processors: try: ready = select.select(processors, [], [])[0] except select.error as e: # If the select call has been interrupted by a signal # just retry if e.args[0] == errno.EINTR: continue raise # For each ready StreamLineProcessor invoke the process() method for stream in ready: eof = stream.process() # Got EOF on this stream if eof: # Remove the stream from the list of valid processors processors.remove(stream) @classmethod def make_logging_handler(cls, level, prefix=None): """ Build a handler function that logs every line it receives. The resulting function logs its input at the specified level with an optional prefix. :param level: The log level to use :param prefix: An optional prefix to prepend to the line :return: handler function """ class_logger = logging.getLogger(cls.__name__) def handler(line): if line: if prefix: class_logger.log(level, "%s%s", prefix, line) else: class_logger.log(level, "%s", line) return handler @staticmethod def make_output_handler(prefix=None): """ Build a handler function which prints every line it receives. The resulting function prints (and log it at INFO level) its input with an optional prefix. :param prefix: An optional prefix to prepend to the line :return: handler function """ # Import the output module inside the function to avoid circular # dependency from barman import output def handler(line): if line: if prefix: output.info("%s%s", prefix, line) else: output.info("%s", line) return handler def enable_signal_forwarding(self, signal_id): """ Enable signal forwarding to the subprocess for a specified signal_id :param signal_id: The signal id to be forwarded """ # Get the current signal handler old_handler = signal.getsignal(signal_id) def _handler(sig, frame): """ This signal handler forward the signal to the subprocess then execute the original handler. """ # Forward the signal to the subprocess if self.pipe: self.pipe.send_signal(signal_id) # If the old handler is callable if callable(old_handler): old_handler(sig, frame) # If we have got a SIGTERM, we must exit elif old_handler == signal.SIG_DFL and signal_id == signal.SIGTERM: sys.exit(128 + signal_id) # Set the signal handler signal.signal(signal_id, _handler) class Rsync(Command): """ This class is a wrapper for the rsync system command, which is used vastly by barman """ def __init__(self, rsync='rsync', args=None, ssh=None, ssh_options=None, bwlimit=None, exclude=None, exclude_and_protect=None, include=None, network_compression=None, path=None, **kwargs): """ :param str rsync: rsync executable name :param list[str]|None args: List of additional argument to aways append :param str ssh: the ssh executable to be used when building the `-e` argument :param list[str] ssh_options: the ssh options to be used when building the `-e` argument :param str bwlimit: optional bandwidth limit :param list[str] exclude: list of file to be excluded from the copy :param list[str] exclude_and_protect: list of file to be excluded from the copy, preserving the destination if exists :param list[str] include: list of files to be included in the copy even if excluded. :param bool network_compression: enable the network compression :param str path: PATH to be used while searching for `cmd` :param bool check: Raise a CommandFailedException if the exit code is not present in `allowed_retval` :param list[int] allowed_retval: List of exit codes considered as a successful termination. """ options = [] if ssh: options += ['-e', full_command_quote(ssh, ssh_options)] if network_compression: options += ['-z'] # Include patterns must be before the exclude ones, because the exclude # patterns actually short-circuit the directory traversal stage # when rsync finds the files to send. if include: for pattern in include: options += ["--include=%s" % (pattern,)] if exclude: for pattern in exclude: options += ["--exclude=%s" % (pattern,)] if exclude_and_protect: for pattern in exclude_and_protect: options += ["--exclude=%s" % (pattern,), "--filter=P_%s" % (pattern,)] if args: options += self._args_for_suse(args) if bwlimit is not None and bwlimit > 0: options += ["--bwlimit=%s" % bwlimit] # By default check is on and the allowed exit code are 0 and 24 if 'check' not in kwargs: kwargs['check'] = True if 'allowed_retval' not in kwargs: kwargs['allowed_retval'] = (0, 24) Command.__init__(self, rsync, args=options, path=path, **kwargs) def _args_for_suse(self, args): """ Mangle args for SUSE compatibility See https://bugzilla.opensuse.org/show_bug.cgi?id=898513 """ # Prepend any argument starting with ':' with a space # Workaround for SUSE rsync issue return [' ' + a if a.startswith(':') else a for a in args] def get_output(self, *args, **kwargs): """ Run the command and return the output and the error (if present) """ # Prepares args for SUSE args = self._args_for_suse(args) # Invoke the base class method return super(Rsync, self).get_output(*args, **kwargs) def from_file_list(self, filelist, src, dst, *args, **kwargs): """ This method copies filelist from src to dst. Returns the return code of the rsync command """ if 'stdin' in kwargs: raise TypeError("from_file_list() doesn't support 'stdin' " "keyword argument") input_string = ('\n'.join(filelist)).encode('UTF-8') _logger.debug("from_file_list: %r", filelist) kwargs['stdin'] = input_string self.get_output('--files-from=-', src, dst, *args, **kwargs) return self.ret class RsyncPgData(Rsync): """ This class is a wrapper for rsync, specialised in sync-ing the Postgres data directory """ def __init__(self, rsync='rsync', args=None, **kwargs): """ Constructor :param str rsync: command to run """ options = ['-rLKpts', '--delete-excluded', '--inplace'] if args: options += args Rsync.__init__(self, rsync, args=options, **kwargs) class PostgreSQLClient(Command): """ Superclass of all the PostgreSQL client commands. """ COMMAND_ALTERNATIVES = None """ Sometimes the name of a command has been changed during the PostgreSQL evolution. I.e. that happened with pg_receivexlog, that has been renamed to pg_receivewal. In that case, we should try using pg_receivewal (the newer auternative) and, if that command doesn't exist, we should try using `pg_receivewal`. This is a list of command names to be used to find the installed command. """ def __init__(self, connection, command, version=None, app_name=None, path=None, **kwargs): """ Constructor :param PostgreSQL connection: an object representing a database connection :param str command: the command to use :param Version version: the command version :param str app_name: the application name to use for the connection :param str path: additional path for executable retrieval """ Command.__init__(self, command, path=path, **kwargs) if version and version >= Version("9.3"): # If version of the client is >= 9.3 we use the connection # string because allows the user to use all the parameters # supported by the libpq library to create a connection conn_string = connection.get_connection_string(app_name) self.args.append("--dbname=%s" % conn_string) else: # 9.2 version doesn't support # connection strings so the 'split' version of the conninfo # option is used instead. conn_params = connection.conn_parameters self.args.append("--host=%s" % conn_params.get('host', None)) self.args.append("--port=%s" % conn_params.get('port', None)) self.args.append("--username=%s" % conn_params.get('user', None)) self.enable_signal_forwarding(signal.SIGINT) self.enable_signal_forwarding(signal.SIGTERM) @classmethod def find_command(cls, path=None): """ Find the active command, given all the alternatives as set in the property named `COMMAND_ALTERNATIVES` in this class. :param str path: The path to use while searching for the command :rtype: Command """ # TODO: Unit tests of this one # To search for an available command, testing if the command # exists in PATH is not sufficient. Debian will install wrappers for # all commands, even if the real command doesn't work. # # I.e. we may have a wrapper for `pg_receivewal` even it PostgreSQL # 10 isn't installed. # # This is an example of what can happen in this case: # # ``` # $ pg_receivewal --version; echo $? # Error: pg_wrapper: pg_receivewal was not found in # /usr/lib/postgresql/9.6/bin # 1 # $ pg_receivexlog --version; echo $? # pg_receivexlog (PostgreSQL) 9.6.3 # 0 # ``` # # That means we should not only ensure the existence of the command, # but we also need to invoke the command to see if it is a shim # or not. # Get the system path if needed if path is None: path = os.getenv('PATH') # If the path is None at this point we have nothing to search if path is None: path = '' # Search the requested executable in every directory present # in path and return a Command object first occurrence that exists, # is executable and runs without errors. for path_entry in path.split(os.path.pathsep): for cmd in cls.COMMAND_ALTERNATIVES: full_path = barman.utils.which(cmd, path_entry) # It doesn't exist try another if not full_path: continue # It exists, let's try invoking it with `--version` to check if # it's real or not. try: command = Command(full_path, path=path, check=True) command("--version") return command except CommandFailedException: # It's only a inactive shim continue # We don't have such a command raise CommandFailedException( 'command not in PATH, tried: %s' % ' '.join(cls.COMMAND_ALTERNATIVES)) @classmethod def get_version_info(cls, path=None): """ Return a dictionary containing all the info about the version of the PostgreSQL client :param str path: the PATH env """ if cls.COMMAND_ALTERNATIVES is None: raise NotImplementedError( "get_version_info cannot be invoked on %s" % cls.__name__) version_info = dict.fromkeys(('full_path', 'full_version', 'major_version'), None) # Get the version string try: command = cls.find_command(path) except CommandFailedException as e: _logger.debug("Error invoking %s: %s", cls.__name__, e) return version_info version_info['full_path'] = command.cmd # Parse the full text version try: full_version = command.out.strip().split()[-1] version_info['full_version'] = Version(full_version) except IndexError: _logger.debug("Error parsing %s version output", version_info['full_path']) return version_info # Extract the major version version_info['major_version'] = Version(barman.utils.simplify_version( full_version)) return version_info class PgBaseBackup(PostgreSQLClient): """ Wrapper class for the pg_basebackup system command """ COMMAND_ALTERNATIVES = ['pg_basebackup'] def __init__(self, connection, destination, command, version=None, app_name=None, bwlimit=None, tbs_mapping=None, immediate=False, check=True, args=None, **kwargs): """ Constructor :param PostgreSQL connection: an object representing a database connection :param str destination: destination directory path :param str command: the command to use :param Version version: the command version :param str app_name: the application name to use for the connection :param str bwlimit: bandwidth limit for pg_basebackup :param Dict[str, str] tbs_mapping: used for tablespace :param bool immediate: fast checkpoint identifier for pg_basebackup :param bool check: check if the return value is in the list of allowed values of the Command obj :param List[str] args: additional arguments """ PostgreSQLClient.__init__( self, connection=connection, command=command, version=version, app_name=app_name, check=check, **kwargs) # Set the backup destination self.args += ['-v', '--no-password', '--pgdata=%s' % destination] if version and version >= Version("10"): # If version of the client is >= 10 it would use # a temporary replication slot by default to keep WALs. # We don't need it because Barman already stores the full # WAL stream, so we disable this feature to avoid wasting one slot. self.args += ['--no-slot'] # The tablespace mapping option is repeated once for each tablespace if tbs_mapping: for (tbs_source, tbs_destination) in tbs_mapping.items(): self.args.append('--tablespace-mapping=%s=%s' % (tbs_source, tbs_destination)) # Only global bandwidth limit is supported if bwlimit is not None and bwlimit > 0: self.args.append("--max-rate=%s" % bwlimit) # Immediate checkpoint if immediate: self.args.append("--checkpoint=fast") # Manage additional args if args: self.args += args class PgReceiveXlog(PostgreSQLClient): """ Wrapper class for pg_receivexlog """ COMMAND_ALTERNATIVES = ["pg_receivewal", "pg_receivexlog"] def __init__(self, connection, destination, command, version=None, app_name=None, synchronous=False, check=True, slot_name=None, args=None, **kwargs): """ Constructor :param PostgreSQL connection: an object representing a database connection :param str destination: destination directory path :param str command: the command to use :param Version version: the command version :param str app_name: the application name to use for the connection :param bool synchronous: request synchronous WAL streaming :param bool check: check if the return value is in the list of allowed values of the Command obj :param str slot_name: the replication slot name to use for the connection :param List[str] args: additional arguments """ PostgreSQLClient.__init__( self, connection=connection, command=command, version=version, app_name=app_name, check=check, **kwargs) self.args += [ "--verbose", "--no-loop", "--no-password", "--directory=%s" % destination] # Add the replication slot name if set in the configuration. if slot_name is not None: self.args.append('--slot=%s' % slot_name) # Request synchronous mode if synchronous: self.args.append('--synchronous') # Manage additional args if args: self.args += args class BarmanSubProcess(object): """ Wrapper class for barman sub instances """ def __init__(self, command=sys.argv[0], subcommand=None, config=None, args=None): """ Build a specific wrapper for all the barman sub-commands, providing an unified interface. :param str command: path to barman :param str subcommand: the barman sub-command :param str config: path to the barman configuration file. :param list[str] args: a list containing the sub-command args like the target server name """ # The config argument is needed when the user explicitly # passes a configuration file, as the child process # must know the configuration file to use. # # The configuration file must always be propagated, # even in case of the default one. if not config: raise CommandFailedException( "No configuration file passed to barman subprocess") # Build the sub-command: # * be sure to run it with the right python interpreter # * pass the current configuration file with -c # * set it quiet with -q self.command = [sys.executable, command, '-c', config, '-q', subcommand] # Handle args for the sub-command (like the server name) if args: self.command += args def execute(self): """ Execute the command and pass the output to the configured handlers """ _logger.debug("BarmanSubProcess: %r", self.command) # Redirect all descriptors to /dev/null devnull = open(os.devnull, 'a+') proc = subprocess.Popen( self.command, preexec_fn=os.setsid, close_fds=True, stdin=devnull, stdout=devnull, stderr=devnull) _logger.debug("BarmanSubProcess: subprocess started. pid: %s", proc.pid) def shell_quote(arg): """ Quote a string argument to be safely included in a shell command line. :param str arg: The script argument :return: The argument quoted """ # This is an excerpt of the Bash manual page, and the same applies for # every Posix compliant shell: # # A non-quoted backslash (\) is the escape character. It preserves # the literal value of the next character that follows, with the # exception of . If a \ pair appears, and the # backslash is not itself quoted, the \ is treated as a # line continuation (that is, it is removed from the input # stream and effectively ignored). # # Enclosing characters in single quotes preserves the literal value # of each character within the quotes. A single quote may not occur # between single quotes, even when pre-ceded by a backslash. # # This means that, as long as the original string doesn't contain any # apostrophe character, it can be safely included between single quotes. # # If a single quote is contained in the string, we must terminate the # string with a quote, insert an apostrophe character escaping it with # a backslash, and then start another string using a quote character. assert arg is not None return "'%s'" % arg.replace("'", "'\\''") def full_command_quote(command, args=None): """ Produce a command with quoted arguments :param str command: the command to be executed :param list[str] args: the command arguments :rtype: str """ if args is not None and len(args) > 0: return "%s %s" % ( command, ' '.join([shell_quote(arg) for arg in args])) else: return command barman-2.3/barman/compression.py0000644000076500000240000002460413134472625017543 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module is responsible to manage the compression features of Barman """ import bz2 import gzip import logging import shutil from abc import ABCMeta, abstractmethod from contextlib import closing from barman.command_wrappers import Command from barman.exceptions import (CommandFailedException, CompressionIncompatibility) from barman.utils import with_metaclass _logger = logging.getLogger(__name__) class CompressionManager(object): def __init__(self, config, path): """ Compression manager """ self.config = config self.path = path def check(self, compression=None): """ This method returns True if the compression specified in the configuration file is present in the register, otherwise False """ if not compression: compression = self.config.compression if compression not in compression_registry: return False return True def get_compressor(self, compression=None): """ Returns a new compressor instance :param str compression: Compression name """ if not compression: compression = self.config.compression # Check if the requested compression mechanism is allowed if self.check(compression): return compression_registry[compression]( config=self.config, compression=compression, path=self.path) else: return None def identify_compression(filename): """ Try to guess the compression algorithm of a file :param filename: the pat of the file to identify :rtype: str """ # TODO: manage multiple decompression methods for the same # compression algorithm (e.g. what to do when gzip is detected? # should we use gzip or pigz?) with open(filename, 'rb') as f: file_start = f.read(MAGIC_MAX_LENGTH) for file_type, cls in sorted(compression_registry.items()): if cls.validate(file_start): return file_type return None class Compressor(with_metaclass(ABCMeta, object)): """ Base class for all the compressors """ MAGIC = None def __init__(self, config, compression, path=None): self.config = config self.compression = compression self.path = path @classmethod def validate(cls, file_start): """ Guess if the first bytes of a file are compatible with the compression implemented by this class :param file_start: a binary string representing the first few bytes of a file :rtype: bool """ return cls.MAGIC and file_start.startswith(cls.MAGIC) @abstractmethod def compress(self, src, dst): """ Abstract Method for compression method :param str src: source file path :param str dst: destination file path """ @abstractmethod def decompress(self, src, dst): """ Abstract method for decompression method :param str src: source file path :param str dst: destination file path """ class CommandCompressor(Compressor): """ Base class for compressors built on external commands """ def __init__(self, config, compression, path=None): super(CommandCompressor, self).__init__( config, compression, path) self._compress = None self._decompress = None def compress(self, src, dst): """ Compress using the specific command defined in the sublcass :param src: source file to compress :param dst: destination of the decompression """ return self._compress(src, dst) def decompress(self, src, dst): """ Decompress using the specific command defined in the sublcass :param src: source file to decompress :param dst: destination of the decompression """ return self._decompress(src, dst) def _build_command(self, pipe_command): """ Build the command string and create the actual Command object :param pipe_command: the command used to compress/decompress :rtype: Command """ command = 'command(){ ' command += pipe_command command += ' > "$2" < "$1"' command += ';}; command' return Command(command, shell=True, check=True, path=self.path) class InternalCompressor(Compressor): """ Base class for compressors built on python libraries """ def compress(self, src, dst): """ Compress using the object defined in the sublcass :param src: source file to compress :param dst: destination of the decompression """ try: with open(src, 'rb') as istream: with closing(self._compressor(dst)) as ostream: shutil.copyfileobj(istream, ostream) except Exception as e: # you won't get more information from the compressors anyway raise CommandFailedException(dict(ret=None, err=str(e), out=None)) return 0 def decompress(self, src, dst): """ Decompress using the object defined in the sublcass :param src: source file to decompress :param dst: destination of the decompression """ try: with closing(self._decompressor(src)) as istream: with open(dst, 'wb') as ostream: shutil.copyfileobj(istream, ostream) except Exception as e: # you won't get more information from the compressors anyway raise CommandFailedException(dict(ret=None, err=str(e), out=None)) return 0 @abstractmethod def _decompressor(self, src): """ Abstract decompressor factory method :param src: source file path :return: a file-like readable decompressor object """ @abstractmethod def _compressor(self, dst): """ Abstract compressor factory method :param dst: destination file path :return: a file-like writable compressor object """ class GZipCompressor(CommandCompressor): """ Predefined compressor with GZip """ MAGIC = b'\x1f\x8b\x08' def __init__(self, config, compression, path=None): super(GZipCompressor, self).__init__( config, compression, path) self._compress = self._build_command('gzip -c') self._decompress = self._build_command('gzip -c -d') class PyGZipCompressor(InternalCompressor): """ Predefined compressor that uses GZip Python libraries """ MAGIC = b'\x1f\x8b\x08' def __init__(self, config, compression, path=None): super(PyGZipCompressor, self).__init__( config, compression, path) # Default compression level used in system gzip utility self._level = -1 # Z_DEFAULT_COMPRESSION constant of zlib def _compressor(self, name): return gzip.GzipFile(name, mode='wb', compresslevel=self._level) def _decompressor(self, name): return gzip.GzipFile(name, mode='rb') class PigzCompressor(CommandCompressor): """ Predefined compressor with Pigz Note that pigz on-disk is the same as gzip, so the MAGIC value of this class is the same """ MAGIC = b'\x1f\x8b\x08' def __init__(self, config, compression, path=None): super(PigzCompressor, self).__init__( config, compression, path) self._compress = self._build_command('pigz -c') self._decompress = self._build_command('pigz -c -d') class BZip2Compressor(CommandCompressor): """ Predefined compressor with BZip2 """ MAGIC = b'\x42\x5a\x68' def __init__(self, config, compression, path=None): super(BZip2Compressor, self).__init__( config, compression, path) self._compress = self._build_command('bzip2 -c') self._decompress = self._build_command('bzip2 -c -d') class PyBZip2Compressor(InternalCompressor): """ Predefined compressor with BZip2 Python libraries """ MAGIC = b'\x42\x5a\x68' def __init__(self, config, compression, path=None): super(PyBZip2Compressor, self).__init__( config, compression, path) # Default compression level used in system gzip utility self._level = 9 def _compressor(self, name): return bz2.BZ2File(name, mode='wb', compresslevel=self._level) def _decompressor(self, name): return bz2.BZ2File(name, mode='rb') class CustomCompressor(CommandCompressor): """ Custom compressor """ def __init__(self, config, compression, path=None): if not config.custom_compression_filter: raise CompressionIncompatibility("custom_compression_filter") if not config.custom_decompression_filter: raise CompressionIncompatibility("custom_decompression_filter") super(CustomCompressor, self).__init__( config, compression, path) self._compress = self._build_command( config.custom_compression_filter) self._decompress = self._build_command( config.custom_decompression_filter) # a dictionary mapping all supported compression schema # to the class implementing it # WARNING: items in this dictionary are extracted using alphabetical order # It's important that gzip and bzip2 are positioned before their variants compression_registry = { 'gzip': GZipCompressor, 'pigz': PigzCompressor, 'bzip2': BZip2Compressor, 'pygzip': PyGZipCompressor, 'pybzip2': PyBZip2Compressor, 'custom': CustomCompressor, } #: The longest string needed to identify a compression schema MAGIC_MAX_LENGTH = max(len(x.MAGIC or '') for x in compression_registry.values()) barman-2.3/barman/config.py0000644000076500000240000007314213134472625016450 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module is responsible for all the things related to Barman configuration, such as parsing configuration file. """ import collections import datetime import inspect import logging.handlers import os import re import sys from glob import iglob from barman import output try: from ConfigParser import ConfigParser, NoOptionError except ImportError: from configparser import ConfigParser, NoOptionError # create a namedtuple object called PathConflict with 'label' and 'server' PathConflict = collections.namedtuple('PathConflict', 'label server') _logger = logging.getLogger(__name__) FORBIDDEN_SERVER_NAMES = ['all'] DEFAULT_USER = 'barman' DEFAULT_LOG_LEVEL = logging.INFO DEFAULT_LOG_FORMAT = "%(asctime)s [%(process)s] %(name)s " \ "%(levelname)s: %(message)s" _TRUE_RE = re.compile(r"""^(true|t|yes|1|on)$""", re.IGNORECASE) _FALSE_RE = re.compile(r"""^(false|f|no|0|off)$""", re.IGNORECASE) _TIME_INTERVAL_RE = re.compile(r""" ^\s* (\d+)\s+(day|month|week)s? # N (day|month|week) with optional 's' \s*$ """, re.IGNORECASE | re.VERBOSE) REUSE_BACKUP_VALUES = ('copy', 'link', 'off') # Possible copy methods for backups (must be all lowercase) BACKUP_METHOD_VALUES = ['rsync', 'postgres'] class CsvOption(set): """ Base class for CSV options. Given a comma delimited string, this class is a list containing the submitted options. Internally, it uses a set in order to avoid option replication. Allowed values for the CSV option are contained in the 'value_list' attribute. The 'conflicts' attribute specifies for any value, the list of values that are prohibited (and thus generate a conflict). If a conflict is found, raises a ValueError exception. """ value_list = [] conflicts = {} def __init__(self, value, key, source): # Invoke parent class init and initialize an empty set super(CsvOption, self).__init__() # Parse not None values if value is not None: self.parse(value, key, source) # Validates the object structure before returning the new instance self.validate(key, source) def parse(self, value, key, source): """ Parses a list of values and correctly assign the set of values (removing duplication) and checking for conflicts. """ if not value: return values_list = value.split(',') for val in sorted(values_list): val = val.strip().lower() if val in self.value_list: # check for conflicting values. if a conflict is # found the option is not valid then, raise exception. if val in self.conflicts and self.conflicts[val] in self: raise ValueError("Invalid configuration value '%s' for " "key %s in %s: cannot contain both " "'%s' and '%s'." "Configuration directive ignored." % (val, key, source, val, self.conflicts[val])) else: # otherwise use parsed value self.add(val) else: # not allowed value, reject the configuration raise ValueError("Invalid configuration value '%s' for " "key %s in %s: Unknown option" % (val, key, source)) def validate(self, key, source): """ Override this method for special validation needs """ def to_json(self): """ Output representation of the obj for JSON serialization The result is a string which can be parsed by the same class """ return ",".join(self) class BackupOptions(CsvOption): """ Extends CsvOption class providing all the details for the backup_options field """ # constants containing labels for allowed values EXCLUSIVE_BACKUP = 'exclusive_backup' CONCURRENT_BACKUP = 'concurrent_backup' EXTERNAL_CONFIGURATION = 'external_configuration' # list holding all the allowed values for the BackupOption class value_list = [EXCLUSIVE_BACKUP, CONCURRENT_BACKUP, EXTERNAL_CONFIGURATION] # map holding all the possible conflicts between the allowed values conflicts = { EXCLUSIVE_BACKUP: CONCURRENT_BACKUP, CONCURRENT_BACKUP: EXCLUSIVE_BACKUP, } class RecoveryOptions(CsvOption): """ Extends CsvOption class providing all the details for the recovery_options field """ # constants containing labels for allowed values GET_WAL = 'get-wal' # list holding all the allowed values for the RecoveryOptions class value_list = [GET_WAL] def parse_boolean(value): """ Parse a string to a boolean value :param str value: string representing a boolean :raises ValueError: if the string is an invalid boolean representation """ if _TRUE_RE.match(value): return True if _FALSE_RE.match(value): return False raise ValueError("Invalid boolean representation (use 'true' or 'false')") def parse_time_interval(value): """ Parse a string, transforming it in a time interval. Accepted format: N (day|month|week)s :param str value: the string to evaluate """ # if empty string or none return none if value is None or value == '': return None result = _TIME_INTERVAL_RE.match(value) # if the string doesn't match, the option is invalid if not result: raise ValueError("Invalid value for a time interval %s" % value) # if the int conversion value = int(result.groups()[0]) unit = result.groups()[1][0].lower() # Calculates the time delta if unit == 'd': time_delta = datetime.timedelta(days=value) elif unit == 'w': time_delta = datetime.timedelta(weeks=value) elif unit == 'm': time_delta = datetime.timedelta(days=(31 * value)) else: # This should never happen raise ValueError("Invalid unit time %s" % unit) return time_delta def parse_reuse_backup(value): """ Parse a string to a valid reuse_backup value. Valid values are "copy", "link" and "off" :param str value: reuse_backup value :raises ValueError: if the value is invalid """ if value is None: return None if value.lower() in REUSE_BACKUP_VALUES: return value.lower() raise ValueError( "Invalid value (use '%s' or '%s')" % ( "', '".join(REUSE_BACKUP_VALUES[:-1]), REUSE_BACKUP_VALUES[-1])) def parse_backup_method(value): """ Parse a string to a valid backup_method value. Valid values are contained in BACKUP_METHOD_VALUES list :param str value: backup_method value :raises ValueError: if the value is invalid """ if value is None: return None if value.lower() in BACKUP_METHOD_VALUES: return value.lower() raise ValueError( "Invalid value (must be one in: '%s')" % ( "', '".join(BACKUP_METHOD_VALUES))) class ServerConfig(object): """ This class represents the configuration for a specific Server instance. """ KEYS = [ 'active', 'archiver', 'archiver_batch_size', 'backup_directory', 'backup_method', 'backup_options', 'bandwidth_limit', 'basebackup_retry_sleep', 'basebackup_retry_times', 'basebackups_directory', 'check_timeout', 'compression', 'conninfo', 'custom_compression_filter', 'custom_decompression_filter', 'description', 'disabled', 'errors_directory', 'immediate_checkpoint', 'incoming_wals_directory', 'last_backup_maximum_age', 'max_incoming_wals_queue', 'minimum_redundancy', 'network_compression', 'parallel_jobs', 'path_prefix', 'post_archive_retry_script', 'post_archive_script', 'post_backup_retry_script', 'post_backup_script', 'pre_archive_retry_script', 'pre_archive_script', 'pre_backup_retry_script', 'pre_backup_script', 'recovery_options', 'retention_policy', 'retention_policy_mode', 'reuse_backup', 'slot_name', 'ssh_command', 'streaming_archiver', 'streaming_archiver_batch_size', 'streaming_archiver_name', 'streaming_backup_name', 'streaming_conninfo', 'streaming_wals_directory', 'tablespace_bandwidth_limit', 'wal_retention_policy', 'wals_directory' ] BARMAN_KEYS = [ 'archiver', 'archiver_batch_size', 'backup_method', 'backup_options', 'bandwidth_limit', 'basebackup_retry_sleep', 'basebackup_retry_times', 'check_timeout', 'compression', 'configuration_files_directory', 'custom_compression_filter', 'custom_decompression_filter', 'immediate_checkpoint', 'last_backup_maximum_age', 'max_incoming_wals_queue', 'minimum_redundancy', 'network_compression', 'parallel_jobs', 'path_prefix', 'post_archive_retry_script', 'post_archive_script', 'post_backup_retry_script', 'post_backup_script', 'pre_archive_retry_script', 'pre_archive_script', 'pre_backup_retry_script', 'pre_backup_script', 'recovery_options', 'retention_policy', 'retention_policy_mode', 'reuse_backup', 'slot_name', 'streaming_archiver', 'streaming_archiver_batch_size', 'streaming_archiver_name', 'streaming_backup_name', 'tablespace_bandwidth_limit', 'wal_retention_policy' ] DEFAULTS = { 'active': 'true', 'archiver': 'off', 'archiver_batch_size': '0', 'backup_directory': '%(barman_home)s/%(name)s', 'backup_method': 'rsync', 'backup_options': '', 'basebackup_retry_sleep': '30', 'basebackup_retry_times': '0', 'basebackups_directory': '%(backup_directory)s/base', 'check_timeout': '30', 'disabled': 'false', 'errors_directory': '%(backup_directory)s/errors', 'immediate_checkpoint': 'false', 'incoming_wals_directory': '%(backup_directory)s/incoming', 'minimum_redundancy': '0', 'network_compression': 'false', 'parallel_jobs': '1', 'recovery_options': '', 'retention_policy_mode': 'auto', 'streaming_archiver': 'off', 'streaming_archiver_batch_size': '0', 'streaming_archiver_name': 'barman_receive_wal', 'streaming_backup_name': 'barman_streaming_backup', 'streaming_conninfo': '%(conninfo)s', 'streaming_wals_directory': '%(backup_directory)s/streaming', 'wal_retention_policy': 'main', 'wals_directory': '%(backup_directory)s/wals' } FIXED = [ 'disabled', ] PARSERS = { 'active': parse_boolean, 'archiver': parse_boolean, 'archiver_batch_size': int, 'backup_method': parse_backup_method, 'backup_options': BackupOptions, 'basebackup_retry_sleep': int, 'basebackup_retry_times': int, 'check_timeout': int, 'disabled': parse_boolean, 'immediate_checkpoint': parse_boolean, 'last_backup_maximum_age': parse_time_interval, 'max_incoming_wals_queue': int, 'network_compression': parse_boolean, 'parallel_jobs': int, 'recovery_options': RecoveryOptions, 'reuse_backup': parse_reuse_backup, 'streaming_archiver': parse_boolean, 'streaming_archiver_batch_size': int, } def invoke_parser(self, key, source, value, new_value): """ Function used for parsing configuration values. If needed, it uses special parsers from the PARSERS map, and handles parsing exceptions. Uses two values (value and new_value) to manage configuration hierarchy (server config overwrites global config). :param str key: the name of the configuration option :param str source: the section that contains the configuration option :param value: the old value of the option if present. :param str new_value: the new value that needs to be parsed :return: the parsed value of a configuration option """ # If the new value is None, returns the old value if new_value is None: return value # If we have a parser for the current key, use it to obtain the # actual value. If an exception is thrown, print a warning and # ignore the value. # noinspection PyBroadException if key in self.PARSERS: parser = self.PARSERS[key] try: # If the parser is a subclass of the CsvOption class # we need a different invocation, which passes not only # the value to the parser, but also the key name # and the section that contains the configuration if inspect.isclass(parser) \ and issubclass(parser, CsvOption): value = parser(new_value, key, source) else: value = parser(new_value) except Exception as e: output.warning("Ignoring invalid configuration value '%s' " "for key %s in %s: %s", new_value, key, source, e) else: value = new_value return value def __init__(self, config, name): self.msg_list = [] self.config = config self.name = name self.barman_home = config.barman_home self.barman_lock_directory = config.barman_lock_directory config.validate_server_config(self.name) for key in ServerConfig.KEYS: value = None # Skip parameters that cannot be configured by users if key not in ServerConfig.FIXED: # Get the setting from the [name] section of config file # A literal None value is converted to an empty string new_value = config.get(name, key, self.__dict__, none_value='') source = '[%s] section' % name value = self.invoke_parser(key, source, value, new_value) # If the setting isn't present in [name] section of config file # check if it has to be inherited from the [barman] section if value is None and key in ServerConfig.BARMAN_KEYS: new_value = config.get('barman', key, self.__dict__, none_value='') source = '[barman] section' value = self.invoke_parser(key, source, value, new_value) # If the setting isn't present in [name] section of config file # and is not inherited from global section use its default # (if present) if value is None and key in ServerConfig.DEFAULTS: new_value = ServerConfig.DEFAULTS[key] % self.__dict__ source = 'DEFAULTS' value = self.invoke_parser(key, source, value, new_value) # An empty string is a None value (bypassing inheritance # from global configuration) if value is not None and value == '' or value == 'None': value = None setattr(self, key, value) def to_json(self): """ Return an equivalent dictionary that can be encoded in json """ json_dict = dict(vars(self)) # remove the reference to main Config object del json_dict['config'] return json_dict def get_bwlimit(self, tablespace=None): """ Return the configured bandwidth limit for the provided tablespace If tablespace is None, it returns the global bandwidth limit :param barman.infofile.Tablespace tablespace: the tablespace to copy :rtype: str """ # Default to global bandwidth limit bwlimit = self.bandwidth_limit if tablespace: # A tablespace can be copied using a per-tablespace bwlimit tablespaces_bw_limit = self.tablespace_bandwidth_limit if (tablespaces_bw_limit and tablespace.name in tablespaces_bw_limit): bwlimit = tablespaces_bw_limit[tablespace.name] return bwlimit class Config(object): """This class represents the barman configuration. Default configuration files are /etc/barman.conf, /etc/barman/barman.conf and ~/.barman.conf for a per-user configuration """ CONFIG_FILES = [ '~/.barman.conf', '/etc/barman.conf', '/etc/barman/barman.conf', ] _QUOTE_RE = re.compile(r"""^(["'])(.*)\1$""") def __init__(self, filename=None): self._config = ConfigParser() if filename: if hasattr(filename, 'read'): self._config.readfp(filename) else: # check for the existence of the user defined file if not os.path.exists(filename): sys.exit("Configuration file '%s' does not exist" % filename) self._config.read(os.path.expanduser(filename)) else: # Check for the presence of configuration files # inside default directories for path in self.CONFIG_FILES: full_path = os.path.expanduser(path) if os.path.exists(full_path) \ and full_path in self._config.read(full_path): filename = full_path break else: sys.exit("Could not find any configuration file at " "default locations.\n" "Check Barman's documentation for more help.") self.config_file = filename self._servers = None self.servers_msg_list = [] self._parse_global_config() def get(self, section, option, defaults=None, none_value=None): """Method to get the value from a given section from Barman configuration """ if not self._config.has_section(section): return None try: value = self._config.get(section, option, raw=False, vars=defaults) if value.lower() == 'none': value = none_value if value is not None: value = self._QUOTE_RE.sub(lambda m: m.group(2), value) return value except NoOptionError: return None def _parse_global_config(self): """ This method parses the global [barman] section """ self.barman_home = self.get('barman', 'barman_home') self.barman_lock_directory = self.get( 'barman', 'barman_lock_directory') or self.barman_home self.user = self.get('barman', 'barman_user') or DEFAULT_USER self.log_file = self.get('barman', 'log_file') self.log_format = self.get( 'barman', 'log_format') or DEFAULT_LOG_FORMAT self.log_level = self.get('barman', 'log_level') or DEFAULT_LOG_LEVEL # save the raw barman section to be compared later in # _is_global_config_changed() method self._global_config = set(self._config.items('barman')) def _is_global_config_changed(self): """Return true if something has changed in global configuration""" return self._global_config != set(self._config.items('barman')) def load_configuration_files_directory(self): """ Read the "configuration_files_directory" option and load all the configuration files with the .conf suffix that lie in that folder """ config_files_directory = self.get('barman', 'configuration_files_directory') if not config_files_directory: return if not os.path.isdir(os.path.expanduser(config_files_directory)): _logger.warn( 'Ignoring the "configuration_files_directory" option as "%s" ' 'is not a directory', config_files_directory) return for cfile in sorted(iglob( os.path.join(os.path.expanduser(config_files_directory), '*.conf'))): filename = os.path.basename(cfile) if os.path.isfile(cfile): # Load a file _logger.debug('Including configuration file: %s', filename) self._config.read(cfile) if self._is_global_config_changed(): msg = "the configuration file %s contains a not empty [" \ "barman] section" % filename _logger.fatal(msg) raise SystemExit("FATAL: %s" % msg) else: # Add an info that a file has been discarded _logger.warn('Discarding configuration file: %s (not a file)', filename) def _populate_servers(self): """ Populate server list from configuration file Also check for paths errors in configuration. If two or more paths overlap in a single server, that server is disabled. If two or more directory paths overlap between different servers an error is raised. """ # Populate servers if self._servers is not None: return self._servers = {} # Cycle all the available configurations sections for section in self._config.sections(): if section == 'barman': # skip global settings continue # Exit if the section has a reserved name if section in FORBIDDEN_SERVER_NAMES: msg = "the reserved word '%s' is not allowed as server name." \ "Please rename it." % section _logger.fatal(msg) raise SystemExit("FATAL: %s" % msg) # Create a ServerConfig object self._servers[section] = ServerConfig(self, section) # Check for conflicting paths in Barman configuration self._check_conflicting_paths() def _check_conflicting_paths(self): """ Look for conflicting paths intra-server and inter-server """ # All paths in configuration servers_paths = {} # Global errors list self.servers_msg_list = [] # Cycle all the available configurations sections for section in sorted(self._config.sections()): if section == 'barman': # skip global settings continue # Paths map section_conf = self._servers[section] config_paths = { 'backup_directory': section_conf.backup_directory, 'basebackups_directory': section_conf.basebackups_directory, 'errors_directory': section_conf.errors_directory, 'incoming_wals_directory': section_conf.incoming_wals_directory, 'streaming_wals_directory': section_conf.streaming_wals_directory, 'wals_directory': section_conf.wals_directory, } # Check for path errors for label, path in sorted(config_paths.items()): # If the path does not conflict with the others, add it to the # paths map real_path = os.path.realpath(path) if real_path not in servers_paths: servers_paths[real_path] = PathConflict(label, section) else: if section == servers_paths[real_path].server: # Internal path error. # Insert the error message into the server.msg_list if real_path == path: self._servers[section].msg_list.append( "Conflicting path: %s=%s conflicts with " "'%s' for server '%s'" % ( label, path, servers_paths[real_path].label, servers_paths[real_path].server)) else: # Symbolic link self._servers[section].msg_list.append( "Conflicting path: %s=%s (symlink to: %s) " "conflicts with '%s' for server '%s'" % ( label, path, real_path, servers_paths[real_path].label, servers_paths[real_path].server)) # Disable the server self._servers[section].disabled = True else: # Global path error. # Insert the error message into the global msg_list if real_path == path: self.servers_msg_list.append( "Conflicting path: " "%s=%s for server '%s' conflicts with " "'%s' for server '%s'" % ( label, path, section, servers_paths[real_path].label, servers_paths[real_path].server)) else: # Symbolic link self.servers_msg_list.append( "Conflicting path: " "%s=%s (symlink to: %s) for server '%s' " "conflicts with '%s' for server '%s'" % ( label, path, real_path, section, servers_paths[real_path].label, servers_paths[real_path].server)) def server_names(self): """This method returns a list of server names""" self._populate_servers() return self._servers.keys() def servers(self): """This method returns a list of server parameters""" self._populate_servers() return self._servers.values() def get_server(self, name): """ Get the configuration of the specified server :param str name: the server name """ self._populate_servers() return self._servers.get(name, None) def validate_global_config(self): """ Validate global configuration parameters """ # Check for the existence of unexpected parameters in the # global section of the configuration file keys = ['barman_home', 'barman_lock_directory', 'barman_user', 'log_file', 'log_level', 'configuration_files_directory'] keys.extend(ServerConfig.KEYS) self._validate_with_keys(self._global_config, keys, 'barman') def validate_server_config(self, server): """ Validate configuration parameters for a specified server :param str server: the server name """ # Check for the existence of unexpected parameters in the # server section of the configuration file self._validate_with_keys(self._config.items(server), ServerConfig.KEYS, server) @staticmethod def _validate_with_keys(config_items, allowed_keys, section): """ Check every config parameter against a list of allowed keys :param config_items: list of tuples containing provided parameters along with their values :param allowed_keys: list of allowed keys :param section: source section (for error reporting) """ for parameter in config_items: # if the parameter name is not in the list of allowed values, # then output a warning name = parameter[0] if name not in allowed_keys: output.warning('Invalid configuration option "%s" in [%s] ' 'section.', name, section) # easy raw config diagnostic with python -m # noinspection PyProtectedMember def _main(): print("Active configuration settings:") r = Config() r.load_configuration_files_directory() for section in r._config.sections(): print("Section: %s" % section) for option in r._config.options(section): print("\t%s = %s " % (option, r.get(section, option))) if __name__ == "__main__": _main() barman-2.3/barman/copy_controller.py0000644000076500000240000012353113136327160020411 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ Copy controller module A copy controller will handle the copy between a series of files and directory, and their final destination. """ import collections import datetime import logging import os.path import re import shutil import signal import tempfile from functools import partial from multiprocessing import Lock, Pool import dateutil.parser import dateutil.tz from barman.command_wrappers import RsyncPgData from barman.exceptions import CommandFailedException, RsyncListFilesFailure from barman.utils import human_readable_timedelta, total_seconds _logger = logging.getLogger(__name__) _logger_lock = Lock() _worker_callable = None """ Global variable containing a callable used to execute the jobs. Initialized by `_init_worker` and used by `_run_worker` function. This variable must be None outside a multiprocessing worker Process. """ # Parallel copy bucket size (10GB) BUCKET_SIZE = (1024 * 1024 * 1024 * 10) def _init_worker(func): """ Store the callable used to execute jobs passed to `_run_worker` function :param callable func: the callable to invoke for every job """ global _worker_callable _worker_callable = func def _run_worker(job): """ Execute a job using the callable set using `_init_worker` function :param _RsyncJob job: the job to be executed """ global _worker_callable assert _worker_callable is not None, \ "Worker has not been initialized with `_init_worker`" # This is the entrypoint of the worker process. Since the KeyboardInterrupt # exceptions is handled by the main process, let's forget about Ctrl-C # here. # When the parent process will receive a KeyboardInterrupt, is will ask # the pool to terminate its workers and then terminate itself. signal.signal(signal.SIGINT, signal.SIG_IGN) return _worker_callable(job) class _RsyncJob(object): """ A job to be executed by a worker Process """ def __init__(self, item_idx, description, id=None, file_list=None, checksum=None): """ :param int item_idx: The index of copy item containing this job :param str description: The description of the job, used for logging :param int id: Job ID (as in bucket) :param list[RsyncCopyController._FileItem] file_list: Path to the file containing the file list :param bool checksum: Whether to force the checksum verification """ self.id = id self.item_idx = item_idx self.description = description self.file_list = file_list self.checksum = checksum # Statistics self.copy_start_time = None self.copy_end_time = None class _FileItem(collections.namedtuple('_FileItem', 'mode size date path')): """ This named tuple is used to store the content each line of the output of a "rsync --list-only" call """ class _RsyncCopyItem(object): """ Internal data object that contains the information about one of the items that have to be copied during a RsyncCopyController run. """ def __init__(self, label, src, dst, exclude=None, exclude_and_protect=None, include=None, is_directory=False, bwlimit=None, reuse=None, item_class=None, optional=False): """ The "label" parameter is meant to be used for error messages and logging. If "src" or "dst" content begin with a ':' character, it is a remote path. Only local paths are supported in "reuse" argument. If "reuse" parameter is provided and is not None, it is used to implement the incremental copy. This only works if "is_directory" is True :param str label: a symbolic name for this item :param str src: source directory. :param str dst: destination directory. :param list[str] exclude: list of patterns to be excluded from the copy. The destination will be deleted if present. :param list[str] exclude_and_protect: list of patterns to be excluded from the copy. The destination will be preserved if present. :param list[str] include: list of patterns to be included in the copy even if excluded. :param bool is_directory: Whether the item points to a directory. :param bwlimit: bandwidth limit to be enforced. (KiB) :param str|None reuse: the reference path for incremental mode. :param str|None item_class: If specified carries a meta information about what the object to be copied is. :param bool optional: Whether a failure copying this object should be treated as a fatal failure. This only works if "is_directory" is False """ self.label = label self.src = src self.dst = dst self.exclude = exclude self.exclude_and_protect = exclude_and_protect self.include = include self.is_directory = is_directory self.bwlimit = bwlimit self.reuse = reuse self.item_class = item_class self.optional = optional # Attributes that will e filled during the analysis self.temp_dir = None self.dir_file = None self.exclude_and_protect_file = None self.safe_list = None self.check_list = None # Statistics self.analysis_start_time = None self.analysis_end_time = None def __str__(self): # Prepare strings for messages formatted_class = self.item_class formatted_name = self.src if self.src.startswith(':'): formatted_class = 'remote ' + self.item_class formatted_name = self.src[1:] formatted_class += ' directory' if self.is_directory else ' file' # Log the operation that is being executed if self.item_class in(RsyncCopyController.PGDATA_CLASS, RsyncCopyController.PGCONTROL_CLASS): return "%s: %s" % ( formatted_class, formatted_name) else: return "%s '%s': %s" % ( formatted_class, self.label, formatted_name) class RsyncCopyController(object): """ Copy a list of files and directory to their final destination. """ # Constants to be used as "item_class" values PGDATA_CLASS = "PGDATA" TABLESPACE_CLASS = "tablespace" PGCONTROL_CLASS = "pg_control" CONFIG_CLASS = "config" # This regular expression is used to parse each line of the output # of a "rsync --list-only" call. This regexp has been tested with any known # version of upstream rsync that is supported (>= 3.0.4) LIST_ONLY_RE = re.compile(""" (?x) # Enable verbose mode ^ # start of the line # capture the mode (es. "-rw-------") (?P[-\w]+) \s+ # size is an integer (?P\d+) \s+ # The date field can have two different form (?P # "2014/06/05 18:00:00" if the sending rsync is compiled # with HAVE_STRFTIME [\d/]+\s+[\d:]+ | # "Thu Jun 5 18:00:00 2014" otherwise \w+\s+\w+\s+\d+\s+[\d:]+\s+\d+ ) \s+ # all the remaining characters are part of filename (?P.+) $ # end of the line """) # This regular expression is used to ignore error messages regarding # vanished files that are not really an error. It is used because # in some cases rsync reports it with exit code 23 which could also mean # a fatal error VANISHED_RE = re.compile(""" (?x) # Enable verbose mode ^ # start of the line ( # files which vanished before rsync start rsync:\ link_stat\ ".+"\ failed:\ No\ such\ file\ or\ directory\ \(2\) | # files which vanished after rsync start file\ has\ vanished:\ ".+" | # files which have been truncated during transfer rsync:\ read\ errors\ mapping\ ".+":\ No\ data\ available\ \(61\) | # final summary rsync\ error:\ .* \(code\ 23\)\ at\ main\.c\(\d+\) \ \[generator=[^\]]+\] ) $ # end of the line """) def __init__(self, path=None, ssh_command=None, ssh_options=None, network_compression=False, reuse_backup=None, safe_horizon=None, exclude=None, retry_times=0, retry_sleep=0, workers=1): """ :param str|None path: the PATH where rsync executable will be searched :param str|None ssh_command: the ssh executable to be used to access remote paths :param list[str]|None ssh_options: list of ssh options to be used to access remote paths :param boolean network_compression: whether to use the network compression :param str|None reuse_backup: if "link" or "copy" enables the incremental copy feature :param datetime.datetime|None safe_horizon: if set, assumes that every files older than it are save to copy without checksum verification. :param list[str]|None exclude: list of patterns to be excluded from the copy :param int retry_times: The number of times to retry a failed operation :param int retry_sleep: Sleep time between two retry :param int workers: The number of parallel copy workers """ super(RsyncCopyController, self).__init__() self.path = path self.ssh_command = ssh_command self.ssh_options = ssh_options self.network_compression = network_compression self.reuse_backup = reuse_backup self.safe_horizon = safe_horizon self.exclude = exclude self.retry_times = retry_times self.retry_sleep = retry_sleep self.workers = workers self.item_list = [] """List of items to be copied""" self.rsync_cache = {} """A cache of RsyncPgData objects""" # Attributes used for progress reporting self.total_steps = None """Total number of steps""" self.current_step = None """Current step number""" self.temp_dir = None """Temp dir used to store the status during the copy""" # Statistics self.jobs_done = None """Already finished jobs list""" self.copy_start_time = None """Copy start time""" self.copy_end_time = None """Copy end time""" def add_directory(self, label, src, dst, exclude=None, exclude_and_protect=None, include=None, bwlimit=None, reuse=None, item_class=None): """ Add a directory that we want to copy. If "src" or "dst" content begin with a ':' character, it is a remote path. Only local paths are supported in "reuse" argument. If "reuse" parameter is provided and is not None, it is used to implement the incremental copy. This only works if "is_directory" is True :param str label: symbolic name to be used for error messages and logging. :param str src: source directory. :param str dst: destination directory. :param list[str] exclude: list of patterns to be excluded from the copy. The destination will be deleted if present. :param list[str] exclude_and_protect: list of patterns to be excluded from the copy. The destination will be preserved if present. :param list[str] include: list of patterns to be included in the copy even if excluded. :param bwlimit: bandwidth limit to be enforced. (KiB) :param str|None reuse: the reference path for incremental mode. :param str item_class: If specified carries a meta information about what the object to be copied is. """ self.item_list.append( _RsyncCopyItem( label=label, src=src, dst=dst, is_directory=True, bwlimit=bwlimit, reuse=reuse, item_class=item_class, optional=False, exclude=exclude, exclude_and_protect=exclude_and_protect, include=include)) def add_file(self, label, src, dst, item_class=None, optional=False): """ Add a file that we want to copy :param str label: symbolic name to be used for error messages and logging. :param str src: source directory. :param str dst: destination directory. :param str item_class: If specified carries a meta information about what the object to be copied is. :param bool optional: Whether a failure copying this object should be treated as a fatal failure. """ self.item_list.append( _RsyncCopyItem( label=label, src=src, dst=dst, is_directory=False, bwlimit=None, reuse=None, item_class=item_class, optional=optional)) def _rsync_factory(self, item): """ Build the RsyncPgData object required for copying the provided item :param _RsyncCopyItem item: information about a copy operation :rtype: RsyncPgData """ # If the object already exists, use it if item in self.rsync_cache: return self.rsync_cache[item] # Prepare the command arguments args = self._reuse_args(item.reuse) # Merge the global exclude with the one into the item object if self.exclude and item.exclude: exclude = self.exclude + item.exclude else: exclude = self.exclude or item.exclude # TODO: remove debug output or use it to progress tracking # By adding a double '--itemize-changes' option, the rsync # output will contain the full list of files that have been # touched, even those that have not changed args.append('--itemize-changes') args.append('--itemize-changes') # Build the rsync object that will execute the copy rsync = RsyncPgData( path=self.path, ssh=self.ssh_command, ssh_options=self.ssh_options, args=args, bwlimit=item.bwlimit, network_compression=self.network_compression, exclude=exclude, exclude_and_protect=item.exclude_and_protect, include=item.include, retry_times=self.retry_times, retry_sleep=self.retry_sleep, retry_handler=partial(self._retry_handler, item) ) self.rsync_cache[item] = rsync return rsync def copy(self): """ Execute the actual copy """ # Store the start time self.copy_start_time = datetime.datetime.now() # Create a temporary directory to hold the file lists. self.temp_dir = tempfile.mkdtemp(suffix='', prefix='barman-') # The following try block is to make sure the temporary directory # will be removed on exit and all the pool workers # have been terminated. pool = None try: # Initialize the counters used by progress reporting self._progress_init() _logger.info("Copy started (safe before %r)", self.safe_horizon) # Execute some preliminary steps for each item to be copied for item in self.item_list: # The initial preparation is necessary only for directories if not item.is_directory: continue # Store the analysis start time item.analysis_start_time = datetime.datetime.now() # Analyze the source and destination directory content _logger.info(self._progress_message( "[global] analyze %s" % item)) self._analyze_directory(item) # Prepare the target directories, removing any unneeded file _logger.info(self._progress_message( "[global] create destination directories and delete " "unknown files for %s" % item)) self._create_dir_and_purge(item) # Store the analysis end time item.analysis_end_time = datetime.datetime.now() # Init the list of jobs done. Every job will be added to this list # once finished. The content will be used to calculate statistics # about the copy process. self.jobs_done = [] # The jobs are executed using a parallel processes pool # Each job is generated by `self._job_generator`, it is executed by # `_run_worker` using `self._execute_job`, which has been set # calling `_init_worker` function during the Pool initialization. pool = Pool(processes=self.workers, initializer=_init_worker, initargs=(self._execute_job,)) for job in pool.imap_unordered(_run_worker, self._job_generator( exclude_classes=[self.PGCONTROL_CLASS])): # Store the finished job for further analysis self.jobs_done.append(job) # The PGCONTROL_CLASS items must always be copied last for job in pool.imap_unordered(_run_worker, self._job_generator( include_classes=[self.PGCONTROL_CLASS])): # Store the finished job for further analysis self.jobs_done.append(job) except KeyboardInterrupt: _logger.info("Copy interrupted by the user (safe before %s)", self.safe_horizon) raise except: _logger.info("Copy failed (safe before %s)", self.safe_horizon) raise else: _logger.info("Copy finished (safe before %s)", self.safe_horizon) finally: # The parent process may have finished naturally or have been # interrupted by an exception (i.e. due to a copy error or # the user pressing Ctrl-C). # At this point we must make sure that all the workers have been # correctly terminated before continuing. if pool: pool.terminate() pool.join() # Clean up the temp dir, any exception raised here is logged # and discarded to not clobber an eventual exception being handled. try: shutil.rmtree(self.temp_dir) except EnvironmentError as e: _logger.error("Error cleaning up '%s' (%s)", self.temp_dir, e) self.temp_dir = None # Store the end time self.copy_end_time = datetime.datetime.now() def _job_generator(self, include_classes=None, exclude_classes=None): """ Generate the jobs to be executed by the workers :param list[str]|None include_classes: If not none, copy only the items which have one of the specified classes. :param list[str]|None exclude_classes: If not none, skip all items which have one of the specified classes. :rtype: iter[_RsyncJob] """ for item_idx, item in enumerate(self.item_list): # Skip items of classes which are not required if include_classes and item.item_class not in include_classes: continue if exclude_classes and item.item_class in exclude_classes: continue # If the item is a directory then copy it in two stages, # otherwise copy it using a plain rsync if item.is_directory: # Copy the safe files using the default rsync algorithm msg = self._progress_message( "[%%s] %%s copy safe files from %s" % item) phase_skipped = True for i, bucket in enumerate( self._fill_buckets(item.safe_list)): phase_skipped = False yield _RsyncJob(item_idx, id=i, description=msg, file_list=bucket, checksum=False) if phase_skipped: _logger.info(msg, 'global', 'skipping') # Copy the check files forcing rsync to verify the checksum msg = self._progress_message( "[%%s] %%s copy files with checksum from %s" % item) phase_skipped = True for i, bucket in enumerate( self._fill_buckets(item.check_list)): phase_skipped = False yield _RsyncJob(item_idx, id=i, description=msg, file_list=bucket, checksum=True) if phase_skipped: _logger.info(msg, 'global', 'skipping') else: # Copy the file using plain rsync msg = self._progress_message("[%%s] %%s copy %s" % item) yield _RsyncJob(item_idx, description=msg) def _fill_buckets(self, file_list): """ Generate buckets for parallel copy :param list[_FileItem] file_list: list of file to transfer :rtype: iter[list[_FileItem]] """ # If there is only one worker, fall back to copying all file at once if self.workers < 2: yield file_list return # Create `self.workers` buckets buckets = [[] for _ in range(self.workers)] bucket_sizes = [0 for _ in range(self.workers)] pos = -1 # Sort the list by size for entry in sorted(file_list, key=lambda item: item.size): # Try to fill the file in a bucket for i in range(self.workers): pos = (pos + 1) % self.workers new_size = bucket_sizes[pos] + entry.size if new_size < BUCKET_SIZE: bucket_sizes[pos] = new_size buckets[pos].append(entry) break else: # All the buckets are filled, so return them all for i in range(self.workers): if len(buckets[i]) > 0: yield buckets[i] # Clear the bucket buckets[i] = [] bucket_sizes[i] = 0 # Put the current file in the first bucket bucket_sizes[0] = entry.size buckets[0].append(entry) pos = 0 # Send all the remaining buckets for i in range(self.workers): if len(buckets[i]) > 0: yield buckets[i] def _execute_job(self, job): """ Execute a `_RsyncJob` in a worker process :type job: _RsyncJob """ item = self.item_list[job.item_idx] if job.id is not None: bucket = 'bucket %s' % job.id else: bucket = 'global' # Build the rsync object required for the copy rsync = self._rsync_factory(item) # Store the start time job.copy_start_time = datetime.datetime.now() # Write in the log that the job is starting with _logger_lock: _logger.info(job.description, bucket, 'starting') if item.is_directory: # A directory item must always have checksum and file_list set assert job.file_list is not None, \ 'A directory item must not have a None `file_list` attribute' assert job.checksum is not None, \ 'A directory item must not have a None `checksum` attribute' # Generate a unique name for the file containing the list of files file_list_path = os.path.join( self.temp_dir, '%s_%s_%s.list' % ( item.label, 'check' if job.checksum else 'safe', os.getpid())) # Write the list, one path per line with open(file_list_path, 'w') as file_list: for entry in job.file_list: assert isinstance(entry, _FileItem), \ "expect %r to be a _FileItem" % entry file_list.write(entry.path + "\n") self._copy(rsync, item.src, item.dst, file_list=file_list_path, checksum=job.checksum) else: # A file must never have checksum and file_list set assert job.file_list is None, \ 'A file item must have a None `file_list` attribute' assert job.checksum is None, \ 'A file item must have a None `checksum` attribute' rsync(item.src, item.dst, allowed_retval=(0, 23, 24)) if rsync.ret == 23: if item.optional: _logger.warning( "Ignoring error reading %s", item) else: raise CommandFailedException(dict( ret=rsync.ret, out=rsync.out, err=rsync.err)) # Store the stop time job.copy_end_time = datetime.datetime.now() # Write in the log that the job is finished with _logger_lock: _logger.info(job.description, bucket, 'finished (duration: %s)' % human_readable_timedelta( job.copy_end_time - job.copy_start_time)) # Return the job to the caller, for statistics purpose return job def _progress_init(self): """ Init counters used by progress logging """ self.total_steps = 0 for item in self.item_list: # Directories require 4 steps, files only one if item.is_directory: self.total_steps += 4 else: self.total_steps += 1 self.current_step = 0 def _progress_message(self, msg): """ Log a message containing the progress :param str msg: the message :return srt: message to log """ self.current_step += 1 return "Copy step %s of %s: %s" % ( self.current_step, self.total_steps, msg) def _reuse_args(self, reuse_directory): """ If reuse_backup is 'copy' or 'link', build the rsync option to enable the reuse, otherwise returns an empty list :param str reuse_directory: the local path with data to be reused :rtype: list[str] """ if self.reuse_backup in ('copy', 'link') and \ reuse_directory is not None: return ['--%s-dest=%s' % (self.reuse_backup, reuse_directory)] else: return [] def _retry_handler(self, item, command, args, kwargs, attempt, exc): """ :param _RsyncCopyItem item: The item that is being processed :param RsyncPgData command: Command object being executed :param list args: command args :param dict kwargs: command kwargs :param int attempt: attempt number (starting from 0) :param CommandFailedException exc: the exception which caused the failure """ _logger.warn("Failure executing rsync on %s (attempt %s)", item, attempt) _logger.warn("Retrying in %s seconds", self.retry_sleep) def _analyze_directory(self, item): """ Analyzes the status of source and destination directories identifying the files that are safe from the point of view of a PostgreSQL backup. The safe_horizon value is the timestamp of the beginning of the older backup involved in copy (as source or destination). Any files updated after that timestamp, must be checked as they could have been modified during the backup - and we do not reply WAL files to update them. The destination directory must exist. If the "safe_horizon" parameter is None, we cannot make any assumptions about what can be considered "safe", so we must check everything with checksums enabled. If "ref" parameter is provided and is not None, it is looked up instead of the "dst" dir. This is useful when we are copying files using '--link-dest' and '--copy-dest' rsync options. In this case, both the "dst" and "ref" dir must exist and the "dst" dir must be empty. If source or destination path begin with a ':' character, it is a remote path. Only local paths are supported in "ref" argument. :param _RsyncCopyItem item: information about a copy operation """ # Build the rsync object required for the analysis rsync = self._rsync_factory(item) # If reference is not set we use dst as reference path ref = item.reuse if ref is None: ref = item.dst # Make sure the ref path ends with a '/' or rsync will add the # last path component to all the returned items during listing if ref[-1] != '/': ref += '/' # Build a hash containing all files present on reference directory. # Directories are not included try: ref_hash = dict(( (item.path, item) for item in self._list_files(rsync, ref) if item.mode[0] != 'd')) except (CommandFailedException, RsyncListFilesFailure) as e: # Here we set ref_hash to None, thus disable the code that marks as # "safe matching" those destination files with different time or # size, even if newer than "safe_horizon". As a result, all files # newer than "safe_horizon" will be checked through checksums. ref_hash = None _logger.error( "Unable to retrieve reference directory file list. " "Using only source file information to decide which files" " need to be copied with checksums enabled: %s" % e) # The 'dir.list' file will contain every directory in the # source tree item.dir_file = os.path.join(self.temp_dir, '%s_dir.list' % item.label) dir_list = open(item.dir_file, 'w+') # The 'protect.list' file will contain a filter rule to protect # each file present in the source tree. It will be used during # the first phase to delete all the extra files on destination. item.exclude_and_protect_file = os.path.join( self.temp_dir, '%s_exclude_and_protect.filter' % item.label) exclude_and_protect_filter = open(item.exclude_and_protect_file, 'w+') # The `safe_list` will contain all items older than # safe_horizon, as well as files that we know rsync will # check anyway due to a difference in mtime or size item.safe_list = [] # The `check_list` will contain all items that need # to be copied with checksum option enabled item.check_list = [] for entry in self._list_files(rsync, item.src): # If item is a directory, we only need to save it in 'dir.list' if entry.mode[0] == 'd': dir_list.write(entry.path + '\n') continue # Add every file in the source path to the list of files # to be protected from deletion ('exclude_and_protect.filter') exclude_and_protect_filter.write('P ' + entry.path + '\n') exclude_and_protect_filter.write('- ' + entry.path + '\n') # If source item is older than safe_horizon, # add it to 'safe.list' if self.safe_horizon and entry.date < self.safe_horizon: item.safe_list.append(entry) continue # If ref_hash is None, it means we failed to retrieve the # destination file list. We assume the only safe way is to # check every file that is older than safe_horizon if ref_hash is None: item.check_list.append(entry) continue # If source file differs by time or size from the matching # destination, rsync will discover the difference in any case. # It is then safe to skip checksum check here. dst_item = ref_hash.get(entry.path, None) if (dst_item is None or dst_item.size != entry.size or dst_item.date != entry.date): item.safe_list.append(entry) continue # All remaining files must be checked with checksums enabled item.check_list.append(entry) # Close all the control files dir_list.close() exclude_and_protect_filter.close() def _create_dir_and_purge(self, item): """ Create destination directories and delete any unknown file :param _RsyncCopyItem item: information about a copy operation """ # Build the rsync object required for the analysis rsync = self._rsync_factory(item) # Create directories and delete any unknown file self._rsync_ignore_vanished_files( rsync, '--recursive', '--delete', '--files-from=%s' % item.dir_file, '--filter', 'merge %s' % item.exclude_and_protect_file, item.src, item.dst, check=True) def _copy(self, rsync, src, dst, file_list, checksum=False): """ The method execute the call to rsync, using as source a a list of files, and adding the the checksum option if required by the caller. :param Rsync rsync: the Rsync object used to retrieve the list of files inside the directories for copy purposes :param str src: source directory :param str dst: destination directory :param str file_list: path to the file containing the sources for rsync :param bool checksum: if checksum argument for rsync is required """ # Build the rsync call args args = ['--files-from=%s' % file_list] if checksum: # Add checksum option if needed args.append('--checksum') self._rsync_ignore_vanished_files(rsync, src, dst, *args, check=True) def _list_files(self, rsync, path): """ This method recursively retrieves a list of files contained in a directory, either local or remote (if starts with ':') :param Rsync rsync: the Rsync object used to retrieve the list :param str path: the path we want to inspect :except CommandFailedException: if rsync call fails :except RsyncListFilesFailure: if rsync output can't be parsed """ _logger.debug("list_files: %r", path) # Use the --no-human-readable option to avoid digit groupings # in "size" field with rsync >= 3.1.0. # Ref: http://ftp.samba.org/pub/rsync/src/rsync-3.1.0-NEWS rsync.get_output('--no-human-readable', '--list-only', '-r', path, check=True) for line in rsync.out.splitlines(): line = line.rstrip() match = self.LIST_ONLY_RE.match(line) if match: mode = match.group('mode') # no exceptions here: the regexp forces 'size' to be an integer size = int(match.group('size')) try: date = dateutil.parser.parse(match.group('date')) date = date.replace(tzinfo=dateutil.tz.tzlocal()) except (TypeError, ValueError): # This should not happen, due to the regexp msg = ("Unable to parse rsync --list-only output line " "(date): '%s'" % line) _logger.exception(msg) raise RsyncListFilesFailure(msg) path = match.group('path') yield _FileItem(mode, size, date, path) else: # This is a hard error, as we are unable to parse the output # of rsync. It can only happen with a modified or unknown # rsync version (perhaps newer than 3.1?) msg = ("Unable to parse rsync --list-only output line: " "'%s'" % line) _logger.error(msg) raise RsyncListFilesFailure(msg) def _rsync_ignore_vanished_files(self, rsync, *args, **kwargs): """ Wrap an Rsync.get_output() call and ignore missing args TODO: when rsync 3.1 will be widespread, replace this with --ignore-missing-args argument :param Rsync rsync: the Rsync object used to execute the copy """ kwargs['allowed_retval'] = (0, 23, 24) rsync.get_output(*args, **kwargs) # If return code is 23 and there is any error which doesn't match # the VANISHED_RE regexp raise an error if rsync.ret == 23 and rsync.err is not None: for line in rsync.err.splitlines(): match = self.VANISHED_RE.match(line.rstrip()) if match: continue else: _logger.error("First rsync error line: %s", line) raise CommandFailedException(dict( ret=rsync.ret, out=rsync.out, err=rsync.err)) return rsync.out, rsync.err def statistics(self): """ Return statistics about the copy object. :rtype: dict """ # This method can only run at the end of a non empty copy assert self.copy_end_time assert self.item_list assert self.jobs_done # Initialise the result calculating the total runtime stat = { 'total_time': total_seconds( self.copy_end_time - self.copy_start_time), 'number_of_workers': self.workers, 'analysis_time_per_item': {}, 'copy_time_per_item': {}, 'serialized_copy_time_per_item': {}, } # Calculate the time spent during the analysis of the items analysis_start = None analysis_end = None for item in self.item_list: # Some items don't require analysis if not item.analysis_end_time: continue # Build a human readable name to refer to an item in the output ident = item.label if (analysis_start is None or analysis_start > item.analysis_start_time): analysis_start = item.analysis_start_time if (analysis_end is None or analysis_end < item.analysis_end_time): analysis_end = item.analysis_end_time stat['analysis_time_per_item'][ident] = total_seconds( item.analysis_end_time - item.analysis_start_time) stat['analysis_time'] = total_seconds(analysis_end - analysis_start) # Calculate the time spent per job # WARNING: this code assumes that every item is copied separately, # so it's strictly tied to the `_job_generator` method code item_data = {} for job in self.jobs_done: # WARNING: the item contained in the job is not the same object # contained in self.item_list, as it has gone through two # pickling/unpickling cycle # Build a human readable name to refer to an item in the output ident = self.item_list[job.item_idx].label # If this is the first time we see this item we just store the # values from the job if ident not in item_data: item_data[ident] = { 'start': job.copy_start_time, 'end': job.copy_end_time, 'total_time': job.copy_end_time - job.copy_start_time } else: data = item_data[ident] if data['start'] > job.copy_start_time: data['start'] = job.copy_start_time if data['end'] < job.copy_end_time: data['end'] = job.copy_end_time data['total_time'] += job.copy_end_time - job.copy_start_time # Calculate the time spent copying copy_start = None copy_end = None serialized_time = datetime.timedelta(0) for ident in item_data: data = item_data[ident] if copy_start is None or copy_start > data['start']: copy_start = data['start'] if copy_end is None or copy_end < data['end']: copy_end = data['end'] stat['copy_time_per_item'][ident] = total_seconds( data['end'] - data['start']) stat['serialized_copy_time_per_item'][ident] = total_seconds( data['total_time']) serialized_time += data['total_time'] # Store the total time spent by copying stat['copy_time'] = total_seconds(copy_end - copy_start) stat['serialized_copy_time'] = total_seconds(serialized_time) return stat barman-2.3/barman/diagnose.py0000644000076500000240000000656113136327160016770 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module represents the barman diagnostic tool. """ import json import logging import sys import barman from barman import fs, output from barman.backup import BackupInfo from barman.exceptions import CommandFailedException, FsOperationFailed from barman.utils import BarmanEncoder _logger = logging.getLogger(__name__) def exec_diagnose(servers, errors_list): """ Diagnostic command: gathers information from backup server and from all the configured servers. Gathered information should be used for support and problems detection :param dict(str,barman.server.Server) servers: list of configured servers :param list errors_list: list of global errors """ # global section. info about barman server diagnosis = {'global': {}, 'servers': {}} # barman global config diagnosis['global']['config'] = dict(barman.__config__._global_config) diagnosis['global']['config']['errors_list'] = errors_list try: command = fs.UnixLocalCommand() # basic system info diagnosis['global']['system_info'] = command.get_system_info() except CommandFailedException as e: diagnosis['global']['system_info'] = {'error': repr(e)} diagnosis['global']['system_info']['barman_ver'] = barman.__version__ # per server section for name in sorted(servers): server = servers[name] if server is None: output.error("Unknown server '%s'" % name) continue # server configuration diagnosis['servers'][name] = {} diagnosis['servers'][name]['config'] = vars(server.config) del diagnosis['servers'][name]['config']['config'] # server system info if server.config.ssh_command: try: command = fs.UnixRemoteCommand( ssh_command=server.config.ssh_command, path=server.path ) diagnosis['servers'][name]['system_info'] = ( command.get_system_info()) except FsOperationFailed: pass # barman statuts information for the server diagnosis['servers'][name]['status'] = server.get_remote_status() # backup list backups = server.get_available_backups(BackupInfo.STATUS_ALL) diagnosis['servers'][name]['backups'] = backups # wal status diagnosis['servers'][name]['wals'] = { 'last_archived_wal_per_timeline': server.backup_manager.get_latest_archived_wals_info(), } # Release any PostgreSQL resource server.close() output.info(json.dumps(diagnosis, sys.stdout, cls=BarmanEncoder, indent=4, sort_keys=True)) barman-2.3/barman/exceptions.py0000644000076500000240000001603413140335171017350 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . class BarmanException(Exception): """ The base class of all other barman exceptions """ class ConfigurationException(BarmanException): """ Base exception for all the Configuration errors """ class CommandException(BarmanException): """ Base exception for all the errors related to the execution of a Command. """ class CompressionException(BarmanException): """ Base exception for all the errors related to the execution of a compression action. """ class PostgresException(BarmanException): """ Base exception for all the errors related to PostgreSQL. """ class BackupException(BarmanException): """ Base exception for all the errors related to the execution of a backup. """ class WALFileException(BarmanException): """ Base exception for all the errors related to WAL files. """ def __str__(self): """ Human readable string representation """ return "%s:%s" % (self.__class__.__name__, self.args[0] if self.args else None) class HookScriptException(BarmanException): """ Base exception for all the errors related to Hook Script execution. """ class LockFileException(BarmanException): """ Base exception for lock related errors """ class DuplicateWalFile(WALFileException): """ A duplicate WAL file has been found """ class MatchingDuplicateWalFile(DuplicateWalFile): """ A duplicate WAL file has been found, but it's identical to the one we already have. """ class SshCommandException(CommandException): """ Error parsing ssh_command parameter """ class UnknownBackupIdException(BackupException): """ The searched backup_id doesn't exists """ class BackupInfoBadInitialisation(BackupException): """ Exception for a bad initialization error """ class CommandFailedException(CommandException): """ Exception representing a failed command """ class CommandMaxRetryExceeded(CommandFailedException): """ A command with retry_times > 0 has exceeded the number of available retry """ def __init__(self, exc): """ :param Exception exc: the last exception raised by the command """ self.exc = exc super(CommandMaxRetryExceeded, self).__init__(*exc.args) class RsyncListFilesFailure(CommandException): """ Failure parsing the output of a "rsync --list-only" command """ class DataTransferFailure(CommandException): """ Used to pass failure details from a data transfer Command """ @classmethod def from_command_error(cls, cmd, e, msg): """ This method build a DataTransferFailure exception and report the provided message to the user (both console and log file) along with the output of the failed command. :param str cmd: The command that failed the transfer :param CommandFailedException e: The exception we are handling :param str msg: a descriptive message on what we are trying to do :return DataTransferFailure: will contain the message provided in msg """ details = msg details += "\n%s error:\n" % cmd details += e.args[0]['out'] details += e.args[0]['err'] return cls(details) class CompressionIncompatibility(CompressionException): """ Exception for compression incompatibility """ class FsOperationFailed(CommandException): """ Exception which represents a failed execution of a command on FS """ class LockFileBusy(LockFileException): """ Raised when a lock file is not free """ class LockFilePermissionDenied(LockFileException): """ Raised when a lock file is not accessible """ class LockFileParsingError(LockFileException): """ Raised when the content of the lockfile is unexpected """ class ConninfoException(ConfigurationException): """ Error for missing or failed parsing of the conninfo parameter (DSN) """ class PostgresConnectionError(PostgresException): """ Error connecting to the PostgreSQL server """ def __str__(self): # Returns the first line if self.args and self.args[0]: return str(self.args[0]).splitlines()[0].strip() else: return '' class PostgresAppNameError(PostgresConnectionError): """ Error setting application name with PostgreSQL server """ class PostgresSuperuserRequired(PostgresException): """ Superuser access is required """ class PostgresIsInRecovery(PostgresException): """ PostgreSQL is in recovery, so no write operations are allowed """ class PostgresUnsupportedFeature(PostgresException): """ Unsupported feature """ class PostgresDuplicateReplicationSlot(PostgresException): """ The creation of a physical replication slot failed because the slot already exists """ class PostgresReplicationSlotsFull(PostgresException): """ The creation of a physical replication slot failed because the all the replication slots have been taken """ class PostgresReplicationSlotInUse(PostgresException): """ The drop of a physical replication slot failed because the replication slots is in use """ class PostgresInvalidReplicationSlot(PostgresException): """ Exception representing a failure during the deletion of a non existent replication slot """ class TimeoutError(CommandException): """ A timeout occurred. """ class ArchiverFailure(WALFileException): """ Exception representing a failure during the execution of the archive process """ class BadXlogSegmentName(WALFileException): """ Exception for a bad xlog name """ class BadHistoryFileContents(WALFileException): """ Exception for a corrupted history file """ class AbortedRetryHookScript(HookScriptException): """ Exception for handling abort of retry hook scripts """ def __init__(self, hook): """ Initialise the exception with hook script info """ self.hook = hook def __str__(self): """ String representation """ return ("Abort '%s_%s' retry hook script (%s, exit code: %d)" % ( self.hook.phase, self.hook.name, self.hook.script, self.hook.exit_status)) barman-2.3/barman/fs.py0000644000076500000240000002764513136327160015615 0ustar mnenciastaff00000000000000# Copyright (C) 2013-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . import logging from barman.command_wrappers import Command, full_command_quote from barman.exceptions import FsOperationFailed _logger = logging.getLogger(__name__) def _str(cmd_out): """ Make a string from the output of a CommandWrapper execution. If input is None returns a literal 'None' string :param cmd_out: String or ByteString to convert :return str: a string """ if hasattr(cmd_out, 'decode') and callable(cmd_out.decode): return cmd_out.decode('utf-8', 'replace') else: return str(cmd_out) class UnixLocalCommand(object): """ This class is a wrapper for local calls for file system operations """ def __init__(self, path=None): # initialize a shell self.internal_cmd = Command(cmd='sh', args=['-c'], path=path) def cmd(self, cmd_name, args=[]): """ Execute a command string, escaping it, if necessary """ return self.internal_cmd(full_command_quote(cmd_name, args)) def get_last_output(self): """ Return the output and the error strings from the last executed command :rtype: tuple[str,str] """ return _str(self.internal_cmd.out), _str(self.internal_cmd.err) def create_dir_if_not_exists(self, dir_path): """ This method recursively creates a directory if not exists If the path exists and is not a directory raise an exception. :param str dir_path: full path for the directory """ _logger.debug('Create directory %s if it does not exists' % dir_path) exists = self.exists(dir_path) if exists: is_dir = self.cmd('test', args=['-d', dir_path]) if is_dir != 0: raise FsOperationFailed( 'A file with the same name already exists') else: return False else: # Make parent directories if needed mkdir_ret = self.cmd('mkdir', args=['-p', dir_path]) if mkdir_ret == 0: return True else: raise FsOperationFailed('mkdir execution failed') def delete_if_exists(self, path): """ This method check for the existence of a path. If it exists, then is removed using a rm -fr command, and returns True. If the command fails an exception is raised. If the path does not exists returns False :param path the full path for the directory """ _logger.debug('Delete path %s if exists' % path) exists = self.exists(path, False) if exists: rm_ret = self.cmd('rm', args=['-fr', path]) if rm_ret == 0: return True else: raise FsOperationFailed('rm execution failed') else: return False def check_directory_exists(self, dir_path): """ Check for the existence of a directory in path. if the directory exists returns true. if the directory does not exists returns false. if exists a file and is not a directory raises an exception :param dir_path full path for the directory """ _logger.debug('Check if directory %s exists' % dir_path) exists = self.exists(dir_path) if exists: is_dir = self.cmd('test', args=['-d', dir_path]) if is_dir != 0: raise FsOperationFailed( 'A file with the same name exists, but is not a directory') else: return True else: return False def check_write_permission(self, dir_path): """ check write permission for barman on a given path. Creates a hidden file using touch, then remove the file. returns true if the file is written and removed without problems raise exception if the creation fails. raise exception if the removal fails. :param dir_path full dir_path for the directory to check """ _logger.debug('Check if directory %s is writable' % dir_path) exists = self.exists(dir_path) if exists: is_dir = self.cmd('test', args=['-d', dir_path]) if is_dir == 0: can_write = self.cmd( 'touch', args=["%s/.barman_write_check" % dir_path]) if can_write == 0: can_remove = self.cmd( 'rm', args=["%s/.barman_write_check" % dir_path]) if can_remove == 0: return True else: raise FsOperationFailed('Unable to remove file') else: raise FsOperationFailed( 'Unable to create write check file') else: raise FsOperationFailed('%s is not a directory' % dir_path) else: raise FsOperationFailed('%s does not exists' % dir_path) def create_symbolic_link(self, src, dst): """ Create a symlink pointing to src named dst. Check src exists, if so, checks that destination does not exists. if src is an invalid folder, raises an exception. if dst already exists, raises an exception. if ln -s command fails raises an exception :param src full path to the source of the symlink :param dst full path for the destination of the symlink """ _logger.debug('Create symbolic link %s -> %s' % (dst, src)) exists = self.exists(src) if exists: exists_dst = self.exists(dst) if not exists_dst: link = self.cmd('ln', args=['-s', src, dst]) if link == 0: return True else: raise FsOperationFailed('ln command failed') else: raise FsOperationFailed('ln destination already exists') else: raise FsOperationFailed('ln source does not exists') def get_system_info(self): """ Gather important system information for 'barman diagnose' command """ result = {} # self.internal_cmd.out can be None. The str() call will ensure it # will be translated to a literal 'None' release = '' if self.cmd("lsb_release", args=['-a']) == 0: release = _str(self.internal_cmd.out).rstrip() elif self.exists('/etc/lsb-release'): self.cmd('cat', args=['/etc/lsb-release']) release = "Ubuntu Linux %s" % _str(self.internal_cmd.out).rstrip() elif self.exists('/etc/debian_version'): self.cmd('cat', args=['/etc/debian_version']) release = "Debian GNU/Linux %s" % _str( self.internal_cmd.out).rstrip() elif self.exists('/etc/redhat-release'): self.cmd('cat', args=['/etc/redhat-release']) release = "RedHat Linux %s" % _str(self.internal_cmd.out).rstrip() elif self.cmd('sw_vers') == 0: release = _str(self.internal_cmd.out).rstrip() result['release'] = release self.cmd('uname', args=['-a']) result['kernel_ver'] = _str(self.internal_cmd.out).rstrip() self.cmd('python', args=['--version', '2>&1']) result['python_ver'] = _str(self.internal_cmd.out).rstrip() self.cmd('rsync', args=['--version', '2>&1']) try: result['rsync_ver'] = _str(self.internal_cmd.out).splitlines( True)[0].rstrip() except IndexError: result['rsync_ver'] = '' self.cmd('ssh', args=['-V', '2>&1']) result['ssh_ver'] = _str(self.internal_cmd.out).rstrip() return result def get_file_content(self, path): """ Retrieve the content of a file If the file doesn't exist or isn't readable, it raises an exception. :param str path: full path to the file to read """ _logger.debug('Reading content of file %s' % path) result = self.exists(path) if not result: raise FsOperationFailed('The %s file does not exist' % path) result = self.cmd('test', args=['-r', path]) if result != 0: raise FsOperationFailed('The %s file is not readable' % path) result = self.cmd('cat', args=['%s', path]) if result != 0: raise FsOperationFailed('Failed to execute "cat \'%s\'"' % path) return self.internal_cmd.out def exists(self, path, dereference=True): """ Check for the existence of a path. :param str path: full path to check :param bool dereference: whether dereference symlinks, defaults to True :return bool: if the file exists or not. """ _logger.debug('check for existence of: %s' % path) options = ['-e', path] if not dereference: options += ['-o', '-L', path] result = self.cmd('test', args=options) return result == 0 def ping(self): """ 'Ping' the server executing the `true` command. :return int: the true cmd result """ _logger.debug('execute the true command') result = self.cmd("true") return result def list_dir_content(self, dir_path, options=[]): """ List the contents of a given directory. :param str dir_path: the path where we want the ls to be executed :param list[str] options: a string containing the options for the ls command :return str: the ls cmd output """ _logger.debug('list the content of a directory') ls_options = [] if options: ls_options += options ls_options.append(dir_path) self.cmd('ls', args=ls_options) return self.internal_cmd.out class UnixRemoteCommand(UnixLocalCommand): """ This class is a wrapper for remote calls for file system operations """ # noinspection PyMissingConstructor def __init__(self, ssh_command, ssh_options=None, path=None): """ Uses the same commands as the UnixLocalCommand but the constructor is overridden and a remote shell is initialized using the ssh_command provided by the user :param str ssh_command: the ssh command provided by the user :param list[str] ssh_options: the options to be passed to SSH :param str path: the path to be used if provided, otherwise the PATH environment variable will be used """ # Ensure that ssh_option is iterable if ssh_options is None: ssh_options = [] if ssh_command is None: raise FsOperationFailed('No ssh command provided') self.internal_cmd = Command(ssh_command, args=ssh_options, path=path, shell=True) try: ret = self.cmd("true") except OSError: raise FsOperationFailed("Unable to execute %s" % ssh_command) if ret != 0: raise FsOperationFailed( "Connection failed using '%s %s' return code %s" % ( ssh_command, ' '.join(ssh_options), ret)) barman-2.3/barman/hooks.py0000644000076500000240000002212013140335167016310 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module contains the logic to run hook scripts """ import logging import time from barman import version from barman.command_wrappers import Command from barman.exceptions import AbortedRetryHookScript, UnknownBackupIdException _logger = logging.getLogger(__name__) class HookScriptRunner(object): def __init__(self, backup_manager, name, phase=None, error=None, retry=False, **extra_env): """ Execute a hook script managing its environment """ self.backup_manager = backup_manager self.name = name self.extra_env = extra_env self.phase = phase self.error = error self.retry = retry self.environment = None self.exit_status = None self.exception = None self.script = None self.reset() def reset(self): """ Reset the status of the class. """ self.environment = dict(self.extra_env) config_file = self.backup_manager.config.config.config_file self.environment.update({ 'BARMAN_VERSION': version.__version__, 'BARMAN_SERVER': self.backup_manager.config.name, 'BARMAN_CONFIGURATION': config_file, 'BARMAN_HOOK': self.name, 'BARMAN_RETRY': str(1 if self.retry else 0), }) if self.error: self.environment['BARMAN_ERROR'] = str(self.error) if self.phase: self.environment['BARMAN_PHASE'] = self.phase script_config_name = "%s_%s" % (self.phase, self.name) else: script_config_name = self.name self.script = getattr(self.backup_manager.config, script_config_name, None) self.exit_status = None self.exception = None def env_from_backup_info(self, backup_info): """ Prepare the environment for executing a script :param BackupInfo backup_info: the backup metadata """ try: previous_backup = self.backup_manager.get_previous_backup( backup_info.backup_id) if previous_backup: previous_backup_id = previous_backup.backup_id else: previous_backup_id = '' except UnknownBackupIdException: previous_backup_id = '' self.environment.update({ 'BARMAN_BACKUP_DIR': backup_info.get_basebackup_directory(), 'BARMAN_BACKUP_ID': backup_info.backup_id, 'BARMAN_PREVIOUS_ID': previous_backup_id, 'BARMAN_STATUS': backup_info.status, 'BARMAN_ERROR': backup_info.error or '', }) def env_from_wal_info(self, wal_info, full_path=None, error=None): """ Prepare the environment for executing a script :param WalFileInfo wal_info: the backup metadata :param str full_path: override wal_info.fullpath() result :param str|Exception error: An error message in case of failure """ self.environment.update({ 'BARMAN_SEGMENT': wal_info.name, 'BARMAN_FILE': str(full_path if full_path is not None else wal_info.fullpath(self.backup_manager.server)), 'BARMAN_SIZE': str(wal_info.size), 'BARMAN_TIMESTAMP': str(wal_info.time), 'BARMAN_COMPRESSION': wal_info.compression or '', 'BARMAN_ERROR': str(error or '') }) def run(self): """ Run a a hook script if configured. This method must never throw any exception """ # noinspection PyBroadException try: if self.script: _logger.debug("Attempt to run %s: %s", self.name, self.script) cmd = Command( self.script, env_append=self.environment, path=self.backup_manager.server.path, shell=True, check=False) self.exit_status = cmd() if self.exit_status != 0: details = "%s returned %d\n" \ "Output details:\n" \ % (self.script, self.exit_status) details += cmd.out details += cmd.err _logger.warning(details) else: _logger.debug("%s returned %d", self.script, self.exit_status) return self.exit_status except Exception as e: _logger.exception('Exception running %s', self.name) self.exception = e return None class RetryHookScriptRunner(HookScriptRunner): """ A 'retry' hook script is a special kind of hook script that Barman tries to run indefinitely until it either returns a SUCCESS or ABORT exit code. Retry hook scripts are executed immediately before (pre) and after (post) the command execution. Standard hook scripts are executed immediately before (pre) and after (post) the retry hook scripts. """ # Failed attempts before sleeping for NAP_TIME seconds ATTEMPTS_BEFORE_NAP = 5 # Short break after a failure (in seconds) BREAK_TIME = 3 # Long break (nap, in seconds) after ATTEMPTS_BEFORE_NAP failures NAP_TIME = 60 # ABORT (and STOP) exit code EXIT_ABORT_STOP = 63 # ABORT (and CONTINUE) exit code EXIT_ABORT_CONTINUE = 62 # SUCCESS exit code EXIT_SUCCESS = 0 def __init__(self, backup_manager, name, phase=None, error=None, **extra_env): super(RetryHookScriptRunner, self).__init__( backup_manager, name, phase, error, retry=True, **extra_env) def run(self): """ Run a a 'retry' hook script, if required by configuration. Barman will retry to run the script indefinitely until it returns a EXIT_SUCCESS, or an EXIT_ABORT_CONTINUE, or an EXIT_ABORT_STOP code. There are BREAK_TIME seconds of sleep between every try. Every ATTEMPTS_BEFORE_NAP failures, Barman will sleep for NAP_TIME seconds. """ # If there is no script, exit if self.script is not None: # Keep track of the number of attempts attempts = 1 while True: # Run the script using the standard hook method (inherited) super(RetryHookScriptRunner, self).run() # Run the script until it returns EXIT_ABORT_CONTINUE, # or an EXIT_ABORT_STOP, or EXIT_SUCCESS if self.exit_status in (self.EXIT_ABORT_CONTINUE, self.EXIT_ABORT_STOP, self.EXIT_SUCCESS): break # Check for the number of attempts if attempts <= self.ATTEMPTS_BEFORE_NAP: attempts += 1 # Take a short break _logger.debug("Retry again in %d seconds", self.BREAK_TIME) time.sleep(self.BREAK_TIME) else: # Reset the attempt number and take a longer nap _logger.debug("Reached %d failures. Take a nap " "then retry again in %d seconds", self.ATTEMPTS_BEFORE_NAP, self.NAP_TIME) attempts = 1 time.sleep(self.NAP_TIME) # Outside the loop check for the exit code. if self.exit_status == self.EXIT_ABORT_CONTINUE: # Warn the user if the script exited with EXIT_ABORT_CONTINUE # Notify EXIT_ABORT_CONTINUE exit status because success and # failures are already managed in the superclass run method _logger.warning("%s was aborted (got exit status %d, " "Barman resumes)", self.script, self.exit_status) elif self.exit_status == self.EXIT_ABORT_STOP: # Log the error and raise AbortedRetryHookScript exception _logger.error("%s was aborted (got exit status %d, " "Barman requested to stop)", self.script, self.exit_status) raise AbortedRetryHookScript(self) return self.exit_status barman-2.3/barman/infofile.py0000644000076500000240000006057713134472625017006 0ustar mnenciastaff00000000000000# Copyright (C) 2013-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . import ast import collections import logging import os import dateutil.parser import dateutil.tz from barman import xlog from barman.compression import identify_compression from barman.exceptions import BackupInfoBadInitialisation # Named tuple representing a Tablespace with 'name' 'oid' and 'location' # as property. Tablespace = collections.namedtuple('Tablespace', 'name oid location') # Named tuple representing a file 'path' with an associated 'file_type' TypedFile = collections.namedtuple('ConfFile', 'file_type path') _logger = logging.getLogger(__name__) def output_tablespace_list(tablespaces): """ Return the literal representation of tablespaces as a Python string :param tablespaces tablespaces: list of Tablespaces objects :return str: Literal representation of tablespaces """ if tablespaces: return repr([tuple(item) for item in tablespaces]) else: return None def load_tablespace_list(string): """ Load the tablespaces as a Python list of namedtuple Uses ast to evaluate information about tablespaces. The returned list is used to create a list of namedtuple :param str string: :return list: list of namedtuple representing all the tablespaces """ obj = ast.literal_eval(string) if obj: return [Tablespace._make(item) for item in obj] else: return None def null_repr(obj): """ Return the literal representation of an object :param object obj: object to represent :return str|None: Literal representation of an object or None """ return repr(obj) if obj else None def load_datetime_tz(time_str): """ Load datetime and ensure the result is timezone-aware. If the parsed timestamp is naive, transform it into a timezone-aware one using the local timezone. :param str time_str: string representing a timestamp :return datetime: the parsed timezone-aware datetime """ # dateutil parser returns naive or tz-aware string depending on the format # of the input string timestamp = dateutil.parser.parse(time_str) # if the parsed timestamp is naive, forces it to local timezone if timestamp.tzinfo is None: timestamp = timestamp.replace(tzinfo=dateutil.tz.tzlocal()) return timestamp class Field(object): def __init__(self, name, dump=None, load=None, default=None, doc=None): """ Field descriptor to be used with a FieldListFile subclass. The resulting field is like a normal attribute with two optional associated function: to_str and from_str The Field descriptor can also be used as a decorator class C(FieldListFile): x = Field('x') @x.dump def x(val): return '0x%x' % val @x.load def x(val): return int(val, 16) :param str name: the name of this attribute :param callable dump: function used to dump the content to a disk :param callable load: function used to reload the content from disk :param default: default value for the field :param str doc: docstring of the filed """ self.name = name self.to_str = dump self.from_str = load self.default = default self.__doc__ = doc # noinspection PyUnusedLocal def __get__(self, obj, objtype=None): if obj is None: return self if not hasattr(obj, '_fields'): obj._fields = {} return obj._fields.setdefault(self.name, self.default) def __set__(self, obj, value): if not hasattr(obj, '_fields'): obj._fields = {} obj._fields[self.name] = value def __delete__(self, obj): raise AttributeError("can't delete attribute") def dump(self, to_str): return type(self)(self.name, to_str, self.from_str, self.__doc__) def load(self, from_str): return type(self)(self.name, self.to_str, from_str, self.__doc__) class FieldListFile(object): __slots__ = ('_fields', 'filename') def __init__(self, **kwargs): """ Represent a predefined set of keys with the associated value. The constructor build the object assigning every keyword argument to the corresponding attribute. If a provided keyword argument doesn't has a corresponding attribute an AttributeError exception is raised. The values provided to the constructor must be of the appropriate type for the corresponding attribute. The constructor will not attempt any validation or conversion on them. This class is meant to be an abstract base class. :raises: AttributeError """ self._fields = {} self.filename = None for name in kwargs: field = getattr(type(self), name, None) if isinstance(field, Field): setattr(self, name, kwargs[name]) else: raise AttributeError('unknown attribute %s' % name) @classmethod def from_meta_file(cls, filename): """ Factory method that read the specified file and build an object with its content. :param str filename: the file to read """ o = cls() o.load(filename) return o def save(self, filename=None, file_object=None): """ Serialize the object to the specified file or file object If a file_object is specified it will be used. If the filename is not specified it uses the one memorized in the filename attribute. If neither the filename attribute and parameter are set a ValueError exception is raised. :param str filename: path of the file to write :param file file_object: a file like object to write in :param str filename: the file to write :raises: ValueError """ if file_object: info = file_object else: filename = filename or self.filename if filename: info = open(filename, 'w') else: info = None if not info: raise ValueError( 'either a valid filename or a file_object must be specified') with info: for name in sorted(vars(type(self))): field = getattr(type(self), name) value = getattr(self, name, None) if isinstance(field, Field): if callable(field.to_str): value = field.to_str(value) info.write("%s=%s\n" % (name, value)) def load(self, filename=None, file_object=None): """ Replaces the current object content with the one deserialized from the provided file. This method set the filename attribute. A ValueError exception is raised if the provided file contains any invalid line. :param str filename: path of the file to read :param file file_object: a file like object to read from :param str filename: the file to read :raises: ValueError """ if file_object: info = file_object elif filename: info = open(filename, 'r') else: raise ValueError( 'either filename or file_object must be specified') # detect the filename if a file_object is passed if not filename and file_object: if hasattr(file_object, 'name'): filename = file_object.name # canonicalize filename if filename: self.filename = os.path.abspath(filename) else: self.filename = None filename = '' # This is only for error reporting with info: for line in info: # skip spaces and comments if line.isspace() or line.rstrip().startswith('#'): continue # parse the line of form "key = value" try: name, value = [x.strip() for x in line.split('=', 1)] except ValueError: raise ValueError('invalid line %s in file %s' % ( line.strip(), filename)) # use the from_str function to parse the value field = getattr(type(self), name, None) if value == 'None': value = None elif isinstance(field, Field) and callable(field.from_str): value = field.from_str(value) setattr(self, name, value) def items(self): """ Return a generator returning a list of (key, value) pairs. If a filed has a dump function defined, it will be used. """ for name in sorted(vars(type(self))): field = getattr(type(self), name) value = getattr(self, name, None) if isinstance(field, Field): if callable(field.to_str): value = field.to_str(value) yield (name, value) def __repr__(self): return "%s(%s)" % ( self.__class__.__name__, ', '.join(['%s=%r' % x for x in self.items()])) class WalFileInfo(FieldListFile): """ Metadata of a WAL file. """ __slots__ = ('orig_filename',) name = Field('name', doc='base name of WAL file') size = Field('size', load=int, doc='WAL file size after compression') time = Field('time', load=float, doc='WAL file modification time ' '(seconds since epoch)') compression = Field('compression', doc='compression type') @classmethod def from_file(cls, filename, default_compression=None, **kwargs): """ Factory method to generate a WalFileInfo from a WAL file. Every keyword argument will override any attribute from the provided file. If a keyword argument doesn't has a corresponding attribute an AttributeError exception is raised. :param str filename: the file to inspect :param str default_compression: the compression to set if the current schema is not identifiable. """ stat = os.stat(filename) kwargs.setdefault('name', os.path.basename(filename)) kwargs.setdefault('size', stat.st_size) kwargs.setdefault('time', stat.st_mtime) if 'compression' not in kwargs: kwargs['compression'] = identify_compression(filename) \ or default_compression obj = cls(**kwargs) obj.filename = "%s.meta" % filename obj.orig_filename = filename return obj def to_xlogdb_line(self): """ Format the content of this object as a xlogdb line. """ return "%s\t%s\t%s\t%s\n" % ( self.name, self.size, self.time, self.compression) @classmethod def from_xlogdb_line(cls, line): """ Parse a line from xlog catalogue :param str line: a line in the wal database to parse :rtype: WalFileInfo """ try: name, size, time, compression = line.split() except ValueError: # Old format compatibility (no compression) compression = None try: name, size, time = line.split() except ValueError: raise ValueError("cannot parse line: %r" % (line,)) # The to_xlogdb_line method writes None values as literal 'None' if compression == 'None': compression = None size = int(size) time = float(time) return cls(name=name, size=size, time=time, compression=compression) def to_json(self): """ Return an equivalent dictionary that can be encoded in json """ return dict(self.items()) def relpath(self): """ Returns the WAL file path relative to the server's wals_directory """ return os.path.join(xlog.hash_dir(self.name), self.name) def fullpath(self, server): """ Returns the WAL file full path :param barman.server.Server server: the server that owns the wal file """ return os.path.join(server.config.wals_directory, self.relpath()) class BackupInfo(FieldListFile): #: Conversion to string EMPTY = 'EMPTY' STARTED = 'STARTED' FAILED = 'FAILED' DONE = 'DONE' STATUS_ALL = (EMPTY, STARTED, DONE, FAILED) STATUS_NOT_EMPTY = (STARTED, DONE, FAILED) STATUS_ARCHIVING = (STARTED, DONE) #: Status according to retention policies OBSOLETE = 'OBSOLETE' VALID = 'VALID' POTENTIALLY_OBSOLETE = 'OBSOLETE*' NONE = '-' RETENTION_STATUS = (OBSOLETE, VALID, POTENTIALLY_OBSOLETE, NONE) version = Field('version', load=int) pgdata = Field('pgdata') # Parse the tablespaces as a literal Python list of namedtuple # Output the tablespaces as a literal Python list of tuple tablespaces = Field('tablespaces', load=load_tablespace_list, dump=output_tablespace_list) # Timeline is an integer timeline = Field('timeline', load=int) begin_time = Field('begin_time', load=load_datetime_tz) begin_xlog = Field('begin_xlog') begin_wal = Field('begin_wal') begin_offset = Field('begin_offset', load=int) size = Field('size', load=int) deduplicated_size = Field('deduplicated_size', load=int) end_time = Field('end_time', load=load_datetime_tz) end_xlog = Field('end_xlog') end_wal = Field('end_wal') end_offset = Field('end_offset', load=int) status = Field('status', default=EMPTY) server_name = Field('server_name') error = Field('error') mode = Field('mode') config_file = Field('config_file') hba_file = Field('hba_file') ident_file = Field('ident_file') included_files = Field('included_files', load=ast.literal_eval, dump=null_repr) backup_label = Field('backup_label', load=ast.literal_eval, dump=null_repr) copy_stats = Field('copy_stats', load=ast.literal_eval, dump=null_repr) xlog_segment_size = Field('xlog_segment_size', load=int, default=xlog.DEFAULT_XLOG_SEG_SIZE) __slots__ = ('server', 'config', 'backup_manager', 'backup_id', 'backup_version') def __init__(self, server, info_file=None, backup_id=None, **kwargs): """ Stores meta information about a single backup :param Server server: :param file,str,None info_file: :param str,None backup_id: :raise BackupInfoBadInitialisation: if the info_file content is invalid or neither backup_info or """ # Initialises the attributes for the object # based on the predefined keys super(BackupInfo, self).__init__(**kwargs) self.server = server self.config = server.config self.backup_manager = self.server.backup_manager self.server_name = self.config.name self.mode = self.backup_manager.mode if backup_id: # Cannot pass both info_file and backup_id if info_file: raise BackupInfoBadInitialisation( 'both info_file and backup_id parameters are set') self.backup_id = backup_id self.filename = self.get_filename() # Check if a backup info file for a given server and a given ID # already exists. If so load the values from the file. if os.path.exists(self.filename): self.load(filename=self.filename) elif info_file: if hasattr(info_file, 'read'): # We have been given a file-like object self.load(file_object=info_file) else: # Just a file name self.load(filename=info_file) self.backup_id = self.detect_backup_id() elif not info_file: raise BackupInfoBadInitialisation( 'backup_id and info_file parameters are both unset') # Manage backup version for new backup structure self.backup_version = 2 try: # the presence of pgdata directory is the marker of version 1 if self.backup_id is not None and os.path.exists( os.path.join(self.get_basebackup_directory(), 'pgdata')): self.backup_version = 1 except Exception as e: _logger.warning("Error detecting backup_version, " "use default: 2. Failure reason: %s", e) def get_required_wal_segments(self): """ Get the list of required WAL segments for the current backup """ return xlog.generate_segment_names( self.begin_wal, self.end_wal, self.version, self.xlog_segment_size) def get_list_of_files(self, target): """ Get the list of files for the current backup """ # Walk down the base backup directory if target in ('data', 'standalone', 'full'): for root, _, files in os.walk(self.get_basebackup_directory()): for f in files: yield os.path.join(root, f) if target in 'standalone': # List all the WAL files for this backup for x in self.get_required_wal_segments(): yield self.server.get_wal_full_path(x) if target in ('wal', 'full'): for wal_info in self.server.get_wal_until_next_backup( self, include_history=True): yield wal_info.fullpath(self.server) def detect_backup_id(self): """ Detect the backup ID from the name of the parent dir of the info file """ if self.filename: return os.path.basename(os.path.dirname(self.filename)) else: return None def get_basebackup_directory(self): """ Get the default filename for the backup.info file based on backup ID and server directory for base backups """ return os.path.join(self.config.basebackups_directory, self.backup_id) def get_data_directory(self, tablespace_oid=None): """ Get path to the backup data dir according with the backup version If tablespace_oid is passed, build the path to the tablespace base directory, according with the backup version :param int tablespace_oid: the oid of a valid tablespace """ # Check if a tablespace oid is passed and if is a valid oid if tablespace_oid is not None and ( self.tablespaces is None or all(str(tablespace_oid) != str(tablespace.oid) for tablespace in self.tablespaces)): raise ValueError("Invalid tablespace OID %s" % tablespace_oid) # Build the requested path according to backup_version value path = [self.get_basebackup_directory()] # Check te version of the backup if self.backup_version == 2: # If an oid has been provided, we are looking for a tablespace if tablespace_oid is not None: # Append the oid to the basedir of the backup path.append(str(tablespace_oid)) else: # Looking for the data dir path.append('data') else: # Backup v1, use pgdata as base path.append('pgdata') # If a oid has been provided, we are looking for a tablespace. if tablespace_oid is not None: # Append the path to pg_tblspc/oid folder inside pgdata path.extend(('pg_tblspc', str(tablespace_oid))) # Return the built path return os.path.join(*path) def get_external_config_files(self): """ Identify all the configuration files that reside outside the PGDATA. Returns a list of TypedFile objects. :rtype: list[TypedFile] """ config_files = [] for file_type in ('config_file', 'hba_file', 'ident_file'): config_file = getattr(self, file_type, None) if config_file: # Consider only those that reside outside of the original # PGDATA directory if config_file.startswith(self.pgdata): _logger.debug("Config file '%s' already in PGDATA", config_file[len(self.pgdata) + 1:]) continue config_files.append(TypedFile(file_type, config_file)) # Check for any include directives in PostgreSQL configuration # Currently, include directives are not supported for files that # reside outside PGDATA. These files must be manually backed up. # Barman will emit a warning and list those files if self.included_files: for included_file in self.included_files: if not included_file.startswith(self.pgdata): config_files.append(TypedFile('include', included_file)) return config_files def get_filename(self): """ Get the default filename for the backup.info file based on backup ID and server directory for base backups """ return os.path.join(self.get_basebackup_directory(), 'backup.info') def set_attribute(self, key, value): """ Set a value for a given key """ setattr(self, key, value) def save(self, filename=None, file_object=None): if not file_object: # Make sure the containing directory exists filename = filename or self.filename dir_name = os.path.dirname(filename) if not os.path.exists(dir_name): os.makedirs(dir_name) super(BackupInfo, self).save(filename=filename, file_object=file_object) def to_dict(self): """ Return the backup_info content as a simple dictionary :return dict: """ result = dict(self.items()) result.update(backup_id=self.backup_id, server_name=self.server_name, mode=self.mode, tablespaces=self.tablespaces, included_files=self.included_files, copy_stats=self.copy_stats) return result def to_json(self): """ Return an equivalent dictionary that uses only json-supported types """ data = self.to_dict() # Convert fields which need special types not supported by json if data.get('tablespaces') is not None: data['tablespaces'] = [list(item) for item in data['tablespaces']] if data.get('begin_time') is not None: data['begin_time'] = data['begin_time'].ctime() if data.get('end_time') is not None: data['end_time'] = data['end_time'].ctime() return data @classmethod def from_json(cls, server, json_backup_info): """ Factory method that builds a BackupInfo object from a json dictionary :param barman.Server server: the server related to the Backup :param dict json_backup_info: the data set containing values from json """ data = dict(json_backup_info) # Convert fields which need special types not supported by json if data.get('tablespaces') is not None: data['tablespaces'] = [Tablespace._make(item) for item in data['tablespaces']] if data.get('begin_time') is not None: data['begin_time'] = load_datetime_tz(data['begin_time']) if data.get('end_time') is not None: data['end_time'] = load_datetime_tz(data['end_time']) # Instantiate a BackupInfo object using the converted fields return cls(server, **data) barman-2.3/barman/lockfile.py0000644000076500000240000002251213134472625016766 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module is the lock manager for Barman """ import errno import fcntl import os import re from barman.exceptions import (LockFileBusy, LockFileParsingError, LockFilePermissionDenied) class LockFile(object): """ Ensures that there is only one process which is running against a specified LockFile. It supports the Context Manager interface, allowing the use in with statements. with LockFile('file.lock') as locked: if not locked: print "failed" else: You can also use exceptions on failures try: with LockFile('file.lock', True): except LockFileBusy, e, file: print "failed to lock %s" % file """ LOCK_PATTERN = None """ If defined in a subclass, it must be a compiled regular expression which matches the lock filename. It must provide named groups for the constructor parameters which produce the same lock name. I.e.: >>> ServerWalReceiveLock('/tmp', 'server-name').filename '/tmp/.server-name-receive-wal.lock' >>> ServerWalReceiveLock.LOCK_PATTERN = re.compile( r'\.(?P.+)-receive-wal\.lock') >>> m = ServerWalReceiveLock.LOCK_PATTERN.match( '.server-name-receive-wal.lock') >>> ServerWalReceiveLock('/tmp', **(m.groupdict())).filename '/tmp/.server-name-receive-wal.lock' """ @classmethod def build_if_matches(cls, path): """ Factory method that creates a lock instance if the path matches the lock filename created by the actual class :param path: the full path of a LockFile :return: """ # If LOCK_PATTERN is not defined always return None if not cls.LOCK_PATTERN: return None # Matches the provided path against LOCK_PATTERN lock_directory = os.path.abspath(os.path.dirname(path)) lock_name = os.path.basename(path) match = cls.LOCK_PATTERN.match(lock_name) if match: # Build the lock object for the provided path return cls(lock_directory, **(match.groupdict())) return None def __init__(self, filename, raise_if_fail=True, wait=False): self.filename = os.path.abspath(filename) self.fd = None self.raise_if_fail = raise_if_fail self.wait = wait def acquire(self, raise_if_fail=None, wait=None): """ Creates and holds on to the lock file. When raise_if_fail, a LockFileBusy is raised if the lock is held by someone else and a LockFilePermissionDenied is raised when the user executing barman have insufficient rights for the creation of a LockFile. Returns True if lock has been successfully acquired, False otherwise. :param bool raise_if_fail: If True raise an exception on failure :param bool wait: If True issue a blocking request :returns bool: whether the lock has been acquired """ if self.fd: return True fd = None # method arguments take precedence on class parameters raise_if_fail = raise_if_fail \ if raise_if_fail is not None else self.raise_if_fail wait = wait if wait is not None else self.wait try: # 384 is 0600 in octal, 'rw-------' fd = os.open(self.filename, os.O_CREAT | os.O_RDWR, 384) flags = fcntl.LOCK_EX if not wait: flags |= fcntl.LOCK_NB fcntl.flock(fd, flags) # Once locked, replace the content of the file os.lseek(fd, 0, os.SEEK_SET) os.write(fd, ("%s\n" % os.getpid()).encode('ascii')) # Truncate the file at the current position os.ftruncate(fd, os.lseek(fd, 0, os.SEEK_CUR)) self.fd = fd return True except (OSError, IOError) as e: if fd: os.close(fd) # let's not leak file descriptors if raise_if_fail: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): raise LockFileBusy(self.filename) elif e.errno == errno.EACCES: raise LockFilePermissionDenied(self.filename) else: raise else: return False def release(self): """ Releases the lock. If the lock is not held by the current process it does nothing. """ if not self.fd: return try: fcntl.flock(self.fd, fcntl.LOCK_UN) os.close(self.fd) except (OSError, IOError): pass self.fd = None def __del__(self): """ Avoid stale lock files. """ self.release() # Contextmanager interface def __enter__(self): return self.acquire() def __exit__(self, exception_type, value, traceback): self.release() def get_owner_pid(self): """ Test whether a lock is already held by a process. Returns the PID of the owner process or None if the lock is available. :rtype: int|None :raises LockFileParsingError: when the lock content is garbled :raises LockFilePermissionDenied: when the lockfile is not accessible """ try: self.acquire(raise_if_fail=True, wait=False) except LockFileBusy: try: # Read the lock content and parse the PID # NOTE: We cannot read it in the self.acquire method to avoid # reading the previous locker PID with open(self.filename, 'r') as file_object: return int(file_object.readline().strip()) except ValueError as e: # This should not happen raise LockFileParsingError(e) # release the lock and return None self.release() return None class GlobalCronLock(LockFile): """ This lock protects cron from multiple executions. Creates a global '.cron.lock' lock file under the given lock_directory. """ def __init__(self, lock_directory): super(GlobalCronLock, self).__init__( os.path.join(lock_directory, '.cron.lock'), raise_if_fail=True) class ServerBackupLock(LockFile): """ This lock protects a server from multiple executions of backup command Creates a '.-backup.lock' lock file under the given lock_directory for the named SERVER. """ def __init__(self, lock_directory, server_name): super(ServerBackupLock, self).__init__( os.path.join(lock_directory, '.%s-backup.lock' % server_name), raise_if_fail=True) class ServerCronLock(LockFile): """ This lock protects a server from multiple executions of cron command Creates a '.-cron.lock' lock file under the given lock_directory for the named SERVER. """ def __init__(self, lock_directory, server_name): super(ServerCronLock, self).__init__( os.path.join(lock_directory, '.%s-cron.lock' % server_name), raise_if_fail=True, wait=False) class ServerXLOGDBLock(LockFile): """ This lock protects a server's xlogdb access Creates a '.-xlogdb.lock' lock file under the given lock_directory for the named SERVER. """ def __init__(self, lock_directory, server_name): super(ServerXLOGDBLock, self).__init__( os.path.join(lock_directory, '.%s-xlogdb.lock' % server_name), raise_if_fail=True, wait=True) class ServerWalArchiveLock(LockFile): """ This lock protects a server from multiple executions of wal-archive command Creates a '.-archive-wal.lock' lock file under the given lock_directory for the named SERVER. """ def __init__(self, lock_directory, server_name): super(ServerWalArchiveLock, self).__init__( os.path.join(lock_directory, '.%s-archive-wal.lock' % server_name), raise_if_fail=True, wait=False) class ServerWalReceiveLock(LockFile): """ This lock protects a server from multiple executions of receive-wal command Creates a '.-receive-wal.lock' lock file under the given lock_directory for the named SERVER. """ # TODO: Implement on the other LockFile subclasses LOCK_PATTERN = re.compile(r'\.(?P.+)-receive-wal\.lock') def __init__(self, lock_directory, server_name): super(ServerWalReceiveLock, self).__init__( os.path.join(lock_directory, '.%s-receive-wal.lock' % server_name), raise_if_fail=True, wait=False) barman-2.3/barman/output.py0000644000076500000240000011364513152223257016541 0ustar mnenciastaff00000000000000# Copyright (C) 2013-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module control how the output of Barman will be rendered """ from __future__ import print_function import datetime import inspect import logging import sys from barman.infofile import BackupInfo from barman.utils import human_readable_timedelta, pretty_size from barman.xlog import diff_lsn __all__ = [ 'error_occurred', 'debug', 'info', 'warning', 'error', 'exception', 'result', 'close_and_exit', 'close', 'set_output_writer', 'AVAILABLE_WRITERS', 'DEFAULT_WRITER', 'ConsoleOutputWriter', 'NagiosOutputWriter', ] #: True if error or exception methods have been called error_occurred = False #: Exit code if error occurred error_exit_code = 1 def _format_message(message, args): """ Format a message using the args list. The result will be equivalent to message % args If args list contains a dictionary as its only element the result will be message % args[0] :param str message: the template string to be formatted :param tuple args: a list of arguments :return: the formatted message :rtype: str """ if len(args) == 1 and isinstance(args[0], dict): return message % args[0] elif len(args) > 0: return message % args else: return message def _put(level, message, *args, **kwargs): """ Send the message with all the remaining positional arguments to the configured output manager with the right output level. The message will be sent also to the logger unless explicitly disabled with log=False No checks are performed on level parameter as this method is meant to be called only by this module. If level == 'exception' the stack trace will be also logged :param str level: :param str message: the template string to be formatted :param tuple args: all remaining arguments are passed to the log formatter :key bool log: whether to log the message :key bool is_error: treat this message as an error """ # handle keyword-only parameters log = kwargs.pop('log', True) is_error = kwargs.pop('is_error', False) if len(kwargs): raise TypeError('%s() got an unexpected keyword argument %r' % (inspect.stack()[1][3], kwargs.popitem()[0])) if is_error: global error_occurred error_occurred = True _writer.error_occurred() # dispatch the call to the output handler getattr(_writer, level)(message, *args) # log the message as originating from caller's caller module if log: exc_info = False if level == 'exception': level = 'error' exc_info = True frm = inspect.stack()[2] mod = inspect.getmodule(frm[0]) logger = logging.getLogger(mod.__name__) log_level = logging.getLevelName(level.upper()) logger.log(log_level, message, *args, **{'exc_info': exc_info}) def _dispatch(obj, prefix, name, *args, **kwargs): """ Dispatch the call to the %(prefix)s_%(name) method of the obj object :param obj: the target object :param str prefix: prefix of the method to be called :param str name: name of the method to be called :param tuple args: all remaining positional arguments will be sent to target :param dict kwargs: all remaining keyword arguments will be sent to target :return: the result of the invoked method :raise ValueError: if the target method is not present """ method_name = "%s_%s" % (prefix, name) handler = getattr(obj, method_name, None) if callable(handler): return handler(*args, **kwargs) else: raise ValueError("The object %r does not have the %r method" % ( obj, method_name)) def is_quiet(): """ Calls the "is_quiet" method, accessing the protected parameter _quiet of the instanced OutputWriter :return bool: the _quiet parameter value """ return _writer.is_quiet() def is_debug(): """ Calls the "is_debug" method, accessing the protected parameter _debug of the instanced OutputWriter :return bool: the _debug parameter value """ return _writer.is_debug() def debug(message, *args, **kwargs): """ Output a message with severity 'DEBUG' :key bool log: whether to log the message """ _put('debug', message, *args, **kwargs) def info(message, *args, **kwargs): """ Output a message with severity 'INFO' :key bool log: whether to log the message """ _put('info', message, *args, **kwargs) def warning(message, *args, **kwargs): """ Output a message with severity 'INFO' :key bool log: whether to log the message """ _put('warning', message, *args, **kwargs) def error(message, *args, **kwargs): """ Output a message with severity 'ERROR'. Also records that an error has occurred unless the ignore parameter is True. :key bool ignore: avoid setting an error exit status (default False) :key bool log: whether to log the message """ # ignore is a keyword-only parameter ignore = kwargs.pop('ignore', False) if not ignore: kwargs.setdefault('is_error', True) _put('error', message, *args, **kwargs) def exception(message, *args, **kwargs): """ Output a message with severity 'EXCEPTION' If raise_exception parameter doesn't evaluate to false raise and exception: - if raise_exception is callable raise the result of raise_exception() - if raise_exception is an exception raise it - else raise the last exception again :key bool ignore: avoid setting an error exit status :key raise_exception: raise an exception after the message has been processed :key bool log: whether to log the message """ # ignore and raise_exception are keyword-only parameters ignore = kwargs.pop('ignore', False) # noinspection PyNoneFunctionAssignment raise_exception = kwargs.pop('raise_exception', None) if not ignore: kwargs.setdefault('is_error', True) _put('exception', message, *args, **kwargs) if raise_exception: if callable(raise_exception): # noinspection PyCallingNonCallable raise raise_exception(message) elif isinstance(raise_exception, BaseException): raise raise_exception else: raise def init(command, *args, **kwargs): """ Initialize the output writer for a given command. :param str command: name of the command are being executed :param tuple args: all remaining positional arguments will be sent to the output processor :param dict kwargs: all keyword arguments will be sent to the output processor """ try: _dispatch(_writer, 'init', command, *args, **kwargs) except ValueError: exception('The %s writer does not support the "%s" command', _writer.__class__.__name__, command) close_and_exit() def result(command, *args, **kwargs): """ Output the result of an operation. :param str command: name of the command are being executed :param tuple args: all remaining positional arguments will be sent to the output processor :param dict kwargs: all keyword arguments will be sent to the output processor """ try: _dispatch(_writer, 'result', command, *args, **kwargs) except ValueError: exception('The %s writer does not support the "%s" command', _writer.__class__.__name__, command) close_and_exit() def close_and_exit(): """ Close the output writer and terminate the program. If an error has been emitted the program will report a non zero return value. """ close() if error_occurred: sys.exit(error_exit_code) else: sys.exit(0) def close(): """ Close the output writer. """ _writer.close() def set_output_writer(new_writer, *args, **kwargs): """ Replace the current output writer with a new one. The new_writer parameter can be a symbolic name or an OutputWriter object :param new_writer: the OutputWriter name or the actual OutputWriter :type: string or an OutputWriter :param tuple args: all remaining positional arguments will be passed to the OutputWriter constructor :param dict kwargs: all remaining keyword arguments will be passed to the OutputWriter constructor """ global _writer _writer.close() if new_writer in AVAILABLE_WRITERS: _writer = AVAILABLE_WRITERS[new_writer](*args, **kwargs) else: _writer = new_writer class ConsoleOutputWriter(object): def __init__(self, debug=False, quiet=False): """ Default output writer that output everything on console. :param bool debug: print debug messages on standard error :param bool quiet: don't print info messages """ self._debug = debug self._quiet = quiet #: Used in check command to hold the check results self.result_check_list = [] #: Used in status command to hold the status results self.result_status_list = [] #: The minimal flag. If set the command must output a single list of #: values. self.minimal = False #: The server is active self.active = True def _out(self, message, args): """ Print a message on standard output """ print(_format_message(message, args), file=sys.stdout) def _err(self, message, args): """ Print a message on standard error """ print(_format_message(message, args), file=sys.stderr) def is_quiet(self): """ Access the quiet property of the OutputWriter instance :return bool: if the writer is quiet or not """ return self._quiet def is_debug(self): """ Access the debug property of the OutputWriter instance :return bool: if the writer is in debug mode or not """ return self._debug def debug(self, message, *args): """ Emit debug. """ if self._debug: self._err('DEBUG: %s' % message, args) def info(self, message, *args): """ Normal messages are sent to standard output """ if not self._quiet: self._out(message, args) def warning(self, message, *args): """ Warning messages are sent to standard error """ self._err('WARNING: %s' % message, args) def error(self, message, *args): """ Error messages are sent to standard error """ self._err('ERROR: %s' % message, args) def exception(self, message, *args): """ Warning messages are sent to standard error """ self._err('EXCEPTION: %s' % message, args) def error_occurred(self): """ Called immediately before any message method when the originating call has is_error=True """ def close(self): """ Close the output channel. Nothing to do for console. """ def result_backup(self, backup_info): """ Render the result of a backup. Nothing to do for console. """ # TODO: evaluate to display something useful here def result_recovery(self, results): """ Render the result of a recovery. """ if len(results['changes']) > 0: self.info("") self.info("IMPORTANT") self.info("These settings have been modified to prevent " "data losses") self.info("") for assertion in results['changes']: self.info("%s line %s: %s = %s", assertion.filename, assertion.line, assertion.key, assertion.value) if len(results['warnings']) > 0: self.info("") self.info("WARNING") self.info("You are required to review the following options" " as potentially dangerous") self.info("") for assertion in results['warnings']: self.info("%s line %s: %s = %s", assertion.filename, assertion.line, assertion.key, assertion.value) if results['missing_files']: # At least one file is missing, warn the user self.info("") self.info("WARNING") self.info("The following configuration files have not been " "saved during backup, hence they have not been " "restored.") self.info("You need to manually restore them " "in order to start the recovered PostgreSQL instance:") self.info("") for file_name in results['missing_files']: self.info(" %s" % file_name) if results['delete_barman_xlog']: self.info("") self.info("After the recovery, please remember to remove the " "\"barman_xlog\" directory") self.info("inside the PostgreSQL data directory.") if results['get_wal']: self.info("") self.info("WARNING: 'get-wal' is in the specified " "'recovery_options'.") self.info("Before you start up the PostgreSQL server, please " "review the recovery.conf file") self.info("inside the target directory. Make sure that " "'restore_command' can be executed by " "the PostgreSQL user.") self.info("") self.info("Your PostgreSQL server has been successfully " "prepared for recovery!") def _record_check(self, server_name, check, status, hint): """ Record the check line in result_check_map attribute This method is for subclass use :param str server_name: the server is being checked :param str check: the check name :param bool status: True if succeeded :param str,None hint: hint to print if not None """ self.result_check_list.append(dict( server_name=server_name, check=check, status=status, hint=hint)) if not status and self.active: global error_occurred error_occurred = True def init_check(self, server_name, active): """ Init the check command :param str server_name: the server we are start listing :param boolean active: The server is active """ self.info("Server %s:" % server_name) self.active = active def result_check(self, server_name, check, status, hint=None): """ Record a server result of a server check and output it as INFO :param str server_name: the server is being checked :param str check: the check name :param bool status: True if succeeded :param str,None hint: hint to print if not None """ self._record_check(server_name, check, status, hint) if hint: self.info("\t%s: %s (%s)" % (check, 'OK' if status else 'FAILED', hint)) else: self.info("\t%s: %s" % (check, 'OK' if status else 'FAILED')) def init_list_backup(self, server_name, minimal=False): """ Init the list-backup command :param str server_name: the server we are start listing :param bool minimal: if true output only a list of backup id """ self.minimal = minimal def result_list_backup(self, backup_info, backup_size, wal_size, retention_status): """ Output a single backup in the list-backup command :param BackupInfo backup_info: backup we are displaying :param backup_size: size of base backup (with the required WAL files) :param wal_size: size of WAL files belonging to this backup (without the required WAL files) :param retention_status: retention policy status """ # If minimal is set only output the backup id if self.minimal: self.info(backup_info.backup_id) return out_list = [ "%s %s - " % (backup_info.server_name, backup_info.backup_id)] if backup_info.status == BackupInfo.DONE: end_time = backup_info.end_time.ctime() out_list.append('%s - Size: %s - WAL Size: %s' % (end_time, pretty_size(backup_size), pretty_size(wal_size))) if backup_info.tablespaces: tablespaces = [("%s:%s" % (tablespace.name, tablespace.location)) for tablespace in backup_info.tablespaces] out_list.append(' (tablespaces: %s)' % ', '.join(tablespaces)) if retention_status: out_list.append(' - %s' % retention_status) else: out_list.append(backup_info.status) self.info(''.join(out_list)) def result_show_backup(self, backup_ext_info): """ Output all available information about a backup in show-backup command The argument has to be the result of a Server.get_backup_ext_info() call :param dict backup_ext_info: a dictionary containing the info to display """ data = dict(backup_ext_info) self.info("Backup %s:", data['backup_id']) self.info(" Server Name : %s", data['server_name']) self.info(" Status : %s", data['status']) if data['status'] == BackupInfo.DONE: self.info(" PostgreSQL Version : %s", data['version']) self.info(" PGDATA directory : %s", data['pgdata']) if data['tablespaces']: self.info(" Tablespaces:") for item in data['tablespaces']: self.info(" %s: %s (oid: %s)", item.name, item.location, item.oid) self.info("") self.info(" Base backup information:") self.info(" Disk usage : %s (%s with WALs)", pretty_size(data['size']), pretty_size(data['size'] + data[ 'wal_size'])) if data['deduplicated_size'] is not None and data['size'] > 0: deduplication_ratio = 1 - (float(data['deduplicated_size']) / data['size']) self.info(" Incremental size : %s (-%s)", pretty_size(data['deduplicated_size']), '{percent:.2%}'.format(percent=deduplication_ratio) ) self.info(" Timeline : %s", data['timeline']) self.info(" Begin WAL : %s", data['begin_wal']) self.info(" End WAL : %s", data['end_wal']) self.info(" WAL number : %s", data['wal_num']) # Output WAL compression ratio for basebackup WAL files if data['wal_compression_ratio'] > 0: self.info(" WAL compression ratio: %s", '{percent:.2%}'.format( percent=data['wal_compression_ratio'])) self.info(" Begin time : %s", data['begin_time']) self.info(" End time : %s", data['end_time']) # If copy statistics are available print a summary copy_stats = data.get('copy_stats') if copy_stats: copy_time = copy_stats.get('copy_time') if copy_time: value = human_readable_timedelta( datetime.timedelta(seconds=copy_time)) # Show analysis time if it is more than a second analysis_time = copy_stats.get('analysis_time') if analysis_time is not None and analysis_time >= 1: value += " + %s startup" % (human_readable_timedelta( datetime.timedelta(seconds=analysis_time))) self.info(" Copy time : %s", value) size = data['deduplicated_size'] or data['size'] value = "%s/s" % pretty_size(size/copy_time) number_of_workers = copy_stats.get('number_of_workers', 1) if number_of_workers > 1: value += " (%s jobs)" % number_of_workers self.info(" Estimated throughput : %s", value) self.info(" Begin Offset : %s", data['begin_offset']) self.info(" End Offset : %s", data['end_offset']) self.info(" Begin LSN : %s", data['begin_xlog']) self.info(" End LSN : %s", data['end_xlog']) self.info("") self.info(" WAL information:") self.info(" No of files : %s", data['wal_until_next_num']) self.info(" Disk usage : %s", pretty_size(data['wal_until_next_size'])) # Output WAL rate if data['wals_per_second'] > 0: self.info(" WAL rate : %0.2f/hour", data['wals_per_second'] * 3600) # Output WAL compression ratio for archived WAL files if data['wal_until_next_compression_ratio'] > 0: self.info( " Compression ratio : %s", '{percent:.2%}'.format( percent=data['wal_until_next_compression_ratio'])) self.info(" Last available : %s", data['wal_last']) if data['children_timelines']: timelines = data['children_timelines'] self.info( " Reachable timelines : %s", ", ".join([str(history.tli) for history in timelines])) self.info("") self.info(" Catalog information:") self.info(" Retention Policy : %s", data['retention_policy_status'] or 'not enforced') self.info(" Previous Backup : %s", data.setdefault('previous_backup_id', 'not available') or '- (this is the oldest base backup)') self.info(" Next Backup : %s", data.setdefault('next_backup_id', 'not available') or '- (this is the latest base backup)') if data['children_timelines']: self.info("") self.info( "WARNING: WAL information is inaccurate due to " "multiple timelines interacting with this backup") else: if data['error']: self.info(" Error: : %s", data['error']) def init_status(self, server_name): """ Init the status command :param str server_name: the server we are start listing """ self.info("Server %s:", server_name) def result_status(self, server_name, status, description, message): """ Record a result line of a server status command and output it as INFO :param str server_name: the server is being checked :param str status: the returned status code :param str description: the returned status description :param str,object message: status message. It will be converted to str """ message = str(message) self.result_status_list.append(dict( server_name=server_name, status=status, description=description, message=message)) self.info("\t%s: %s", description, message) def init_replication_status(self, server_name, minimal=False): """ Init the 'standby-status' command :param str server_name: the server we are start listing :param str minimal: minimal output """ self.minimal = minimal def result_replication_status(self, server_name, target, server_lsn, standby_info): """ Record a result line of a server status command and output it as INFO :param str server_name: the replication server :param str target: all|hot-standby|wal-streamer :param str server_lsn: server's current lsn :param StatReplication standby_info: status info of a standby """ if target == 'hot-standby': title = 'hot standby servers' elif target == 'wal-streamer': title = 'WAL streamers' else: title = 'streaming clients' if self.minimal: # Minimal output if server_lsn: # current lsn from the master self.info("%s for master '%s' (LSN @ %s):", title.capitalize(), server_name, server_lsn) else: # We are connected to a standby self.info("%s for slave '%s':", title.capitalize(), server_name) else: # Full output self.info("Status of %s for server '%s':", title, server_name) # current lsn from the master if server_lsn: self.info(" Current LSN on master: %s", server_lsn) if standby_info is not None and not len(standby_info): self.info(" No %s attached", title) return # Minimal output if self.minimal: n = 1 for standby in standby_info: if not standby.replay_lsn: # WAL streamer self.info(" %s. W) %s@%s S:%s W:%s P:%s AN:%s", n, standby.usename, standby.client_addr or 'socket', standby.sent_lsn, standby.write_lsn, standby.sync_priority, standby.application_name) else: # Standby self.info(" %s. %s) %s@%s S:%s F:%s R:%s P:%s AN:%s", n, standby.sync_state[0].upper(), standby.usename, standby.client_addr or 'socket', standby.sent_lsn, standby.flush_lsn, standby.replay_lsn, standby.sync_priority, standby.application_name) n += 1 else: n = 1 self.info(" Number of %s: %s", title, len(standby_info)) for standby in standby_info: self.info("") # Calculate differences in bytes sent_diff = diff_lsn(standby.sent_lsn, standby.current_lsn) write_diff = diff_lsn(standby.write_lsn, standby.current_lsn) flush_diff = diff_lsn(standby.flush_lsn, standby.current_lsn) replay_diff = diff_lsn(standby.replay_lsn, standby.current_lsn) # Determine the sync stage of the client sync_stage = None if not standby.replay_lsn: client_type = 'WAL streamer' max_level = 3 else: client_type = 'standby' max_level = 5 # Only standby can replay WAL info if replay_diff == 0: sync_stage = '5/5 Hot standby (max)' elif flush_diff == 0: sync_stage = '4/5 2-safe' # remote flush # If not yet done, set the sync stage if not sync_stage: if write_diff == 0: sync_stage = '3/%s Remote write' % max_level elif sent_diff == 0: sync_stage = '2/%s WAL Sent (min)' % max_level else: sync_stage = '1/%s 1-safe' % max_level # Synchronous standby if getattr(standby, 'sync_priority', None) > 0: self.info(" %s. #%s %s %s", n, standby.sync_priority, standby.sync_state.capitalize(), client_type) # Asynchronous standby else: self.info(" %s. %s %s", n, standby.sync_state.capitalize(), client_type) self.info(" Application name: %s", standby.application_name) self.info(" Sync stage : %s", sync_stage) if getattr(standby, 'client_addr', None): self.info(" Communication : TCP/IP") self.info(" IP Address : %s " "/ Port: %s / Host: %s", standby.client_addr, standby.client_port, standby.client_hostname or '-') else: self.info(" Communication : Unix domain socket") self.info(" User name : %s", standby.usename) self.info(" Current state : %s (%s)", standby.state, standby.sync_state) if getattr(standby, 'slot_name', None): self.info(" Replication slot: %s", standby.slot_name) self.info(" WAL sender PID : %s", standby.pid) self.info(" Started at : %s", standby.backend_start) if getattr(standby, 'backend_xmin', None): self.info(" Standby's xmin : %s", standby.backend_xmin or '-') if getattr(standby, 'sent_lsn', None): self.info(" Sent LSN : %s (diff: %s)", standby.sent_lsn, pretty_size(sent_diff)) if getattr(standby, 'write_lsn', None): self.info(" Write LSN : %s (diff: %s)", standby.write_lsn, pretty_size(write_diff)) if getattr(standby, 'flush_lsn', None): self.info(" Flush LSN : %s (diff: %s)", standby.flush_lsn, pretty_size(flush_diff)) if getattr(standby, 'replay_lsn', None): self.info(" Replay LSN : %s (diff: %s)", standby.replay_lsn, pretty_size(replay_diff)) n += 1 def init_list_server(self, server_name, minimal=False): """ Init the list-server command :param str server_name: the server we are start listing """ self.minimal = minimal def result_list_server(self, server_name, description=None): """ Output a result line of a list-server command :param str server_name: the server is being checked :param str,None description: server description if applicable """ if self.minimal or not description: self.info("%s", server_name) else: self.info("%s - %s", server_name, description) def init_show_server(self, server_name): """ Init the show-server command output method :param str server_name: the server we are displaying """ self.info("Server %s:" % server_name) def result_show_server(self, server_name, server_info): """ Output the results of the show-server command :param str server_name: the server we are displaying :param dict server_info: a dictionary containing the info to display """ for status, message in sorted(server_info.items()): self.info("\t%s: %s", status, message) class NagiosOutputWriter(ConsoleOutputWriter): """ Nagios output writer. This writer doesn't output anything to console. On close it writes a nagios-plugin compatible status """ def _out(self, message, args): """ Do not print anything on standard output """ def _err(self, message, args): """ Do not print anything on standard error """ def close(self): """ Display the result of a check run as expected by Nagios. Also set the exit code as 2 (CRITICAL) in case of errors """ global error_occurred, error_exit_code # List of all servers that have been checked servers = [] # List of servers reporting issues issues = [] for item in self.result_check_list: # Keep track of all the checked servers if item['server_name'] not in servers: servers.append(item['server_name']) # Keep track of the servers with issues if not item['status'] and item['server_name'] not in issues: issues.append(item['server_name']) # Global error (detected at configuration level) if len(issues) == 0 and error_occurred: print("BARMAN CRITICAL - Global configuration errors") error_exit_code = 2 return if len(issues) > 0: fail_summary = [] details = [] for server in issues: # Join all the issues for a server. Output format is in the # form: # " FAILED: , ... " # All strings will be concatenated into the $SERVICEOUTPUT$ # macro of the Nagios output server_fail = "%s FAILED: %s" % ( server, ", ".join([ item['check'] for item in self.result_check_list if item['server_name'] == server and not item['status'] ])) fail_summary.append(server_fail) # Prepare an array with the detailed output for # the $LONGSERVICEOUTPUT$ macro of the Nagios output # line format: # .: FAILED # .: FAILED (Hint if present) # : FAILED # ..... for issue in self.result_check_list: if issue['server_name'] == server and not issue['status']: fail_detail = "%s.%s: FAILED" % (server, issue['check']) if issue['hint']: fail_detail += " (%s)" % issue['hint'] details.append(fail_detail) # Append the summary of failures to the first line of the output # using * as delimiter if len(servers) == 1: print("BARMAN CRITICAL - server %s has issues * %s" % (servers[0], " * ".join(fail_summary))) else: print("BARMAN CRITICAL - %d server out of %d have issues * " "%s" % (len(issues), len(servers), " * ".join(fail_summary))) # add the detailed list to the output for issue in details: print(issue) error_exit_code = 2 else: # No issues, all good! # Display the output message for a single server check if len(servers) == 1: print("BARMAN OK - Ready to serve the Espresso backup " "for %s" % (servers[0])) else: # Display the output message for several servers, using # '*' as delimiter print("BARMAN OK - Ready to serve the Espresso backup " "for %d server(s) * %s" % ( len(servers), " * ".join([server for server in servers]))) #: This dictionary acts as a registry of available OutputWriters AVAILABLE_WRITERS = { 'console': ConsoleOutputWriter, # nagios is not registered as it isn't a general purpose output writer # 'nagios': NagiosOutputWriter, } #: The default OutputWriter DEFAULT_WRITER = 'console' #: the current active writer. Initialized according DEFAULT_WRITER on load _writer = AVAILABLE_WRITERS[DEFAULT_WRITER]() barman-2.3/barman/postgres.py0000644000076500000240000015052513152223257017045 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module represents the interface towards a PostgreSQL server. """ import atexit import logging from abc import ABCMeta import psycopg2 from psycopg2.errorcodes import (DUPLICATE_OBJECT, OBJECT_IN_USE, UNDEFINED_OBJECT) from psycopg2.extensions import STATUS_IN_TRANSACTION from psycopg2.extras import DictCursor, NamedTupleCursor from barman.exceptions import (ConninfoException, PostgresAppNameError, PostgresConnectionError, PostgresDuplicateReplicationSlot, PostgresException, PostgresInvalidReplicationSlot, PostgresIsInRecovery, PostgresReplicationSlotInUse, PostgresReplicationSlotsFull, PostgresSuperuserRequired, PostgresUnsupportedFeature) from barman.infofile import Tablespace from barman.remote_status import RemoteStatusMixin from barman.utils import simplify_version, with_metaclass from barman.xlog import DEFAULT_XLOG_SEG_SIZE # This is necessary because the CONFIGURATION_LIMIT_EXCEEDED constant # has been added in psycopg2 2.5, but Barman supports version 2.4.2+ so # in case of import error we declare a constant providing the correct value. try: from psycopg2.errorcodes import CONFIGURATION_LIMIT_EXCEEDED except ImportError: CONFIGURATION_LIMIT_EXCEEDED = '53400' _logger = logging.getLogger(__name__) _live_connections = [] """ List of connections to be closed at the interpreter shutdown """ @atexit.register def _atexit(): """ Ensure that all the connections are correctly closed at interpreter shutdown """ # Take a copy of the list because the conn.close() method modify it for conn in list(_live_connections): _logger.warn( "Forcing %s cleanup during process shut down.", conn.__class__.__name__) conn.close() class PostgreSQL(with_metaclass(ABCMeta, RemoteStatusMixin)): """ This abstract class represents a generic interface to a PostgreSQL server. """ CHECK_QUERY = 'SELECT 1' def __init__(self, config, conninfo): """ Abstract base class constructor for PostgreSQL interface. :param barman.config.ServerConfig config: the server configuration :param str conninfo: Connection information (aka DSN) """ super(PostgreSQL, self).__init__() assert conninfo self.config = config self.conninfo = conninfo self._conn = None self.allow_reconnect = True # Build a dictionary with connection info parameters # This is mainly used to speed up search in conninfo try: self.conn_parameters = self.parse_dsn(conninfo) except (ValueError, TypeError) as e: _logger.debug(e) raise ConninfoException('Cannot connect to postgres: "%s" ' 'is not a valid connection string' % conninfo) @staticmethod def parse_dsn(dsn): """ Parse connection parameters from 'conninfo' :param str dsn: Connection information (aka DSN) :rtype: dict[str,str] """ # TODO: this might be made more robust in the future return dict(x.split('=', 1) for x in dsn.split()) @staticmethod def encode_dsn(parameters): """ Build a connection string from a dictionary of connection parameters :param dict[str,str] parameters: Connection parameters :rtype: str """ # TODO: this might be made more robust in the future return ' '.join( ["%s=%s" % (k, v) for k, v in sorted(parameters.items())]) def get_connection_string(self, application_name=None): """ Return the connection string, adding the application_name parameter if requested, unless already defined by user in the connection string :param str application_name: the application_name to add :return str: the connection string """ conn_string = self.conninfo # check if the application name is already defined by user if application_name and 'application_name' not in self.conn_parameters: # Then add the it to the connection string conn_string += ' application_name=%s' % application_name return conn_string def connect(self): """ Generic function for Postgres connection (using psycopg2) """ if not self._check_connection(): try: self._conn = psycopg2.connect(self.conninfo) # If psycopg2 fails to connect to the host, # raise the appropriate exception except psycopg2.DatabaseError as e: raise PostgresConnectionError(str(e).strip()) # Register the connection to the list of live connections _live_connections.append(self) return self._conn def _check_connection(self): """ Return false if the connection is broken :rtype: bool """ # If the connection is not present return False if not self._conn: return False # Check if the connection works by running 'SELECT 1' cursor = None try: cursor = self._conn.cursor() cursor.execute(self.CHECK_QUERY) except psycopg2.DatabaseError: # Connection is broken, so we need to reconnect self.close() # Raise an error if reconnect is not allowed if not self.allow_reconnect: raise PostgresConnectionError( "Connection lost, reconnection not allowed") return False finally: if cursor: cursor.close() return True def close(self): """ Close the connection to PostgreSQL """ if self._conn: # If the connection is still alive, rollback and close it if not self._conn.closed: if self._conn.status == STATUS_IN_TRANSACTION: self._conn.rollback() self._conn.close() # Remove the connection from the live connections list self._conn = None _live_connections.remove(self) def _cursor(self, *args, **kwargs): """ Return a cursor """ conn = self.connect() return conn.cursor(*args, **kwargs) @property def server_version(self): """ Version of PostgreSQL (returned by psycopg2) """ conn = self.connect() return conn.server_version @property def server_txt_version(self): """ Human readable version of PostgreSQL (calculated from server_version) :rtype: str|None """ try: conn = self.connect() major = int(conn.server_version / 10000) minor = int(conn.server_version / 100 % 100) patch = int(conn.server_version % 100) if major < 10: return "%d.%d.%d" % (major, minor, patch) if minor != 0: _logger.warning( "Unexpected non zero minor version %s in %s", minor, conn.server_version) return "%d.%d" % (major, patch) except PostgresConnectionError as e: _logger.debug("Error retrieving PostgreSQL version: %s", str(e).strip()) return None @property def server_major_version(self): """ PostgreSQL major version (calculated from server_txt_version) :rtype: str|None """ result = self.server_txt_version if result is not None: return simplify_version(result) return None class StreamingConnection(PostgreSQL): """ This class represents a streaming connection to a PostgreSQL server. """ CHECK_QUERY = 'IDENTIFY_SYSTEM' def __init__(self, config): """ Streaming connection constructor :param barman.config.ServerConfig config: the server configuration """ if config.streaming_conninfo is None: raise ConninfoException( "Missing 'streaming_conninfo' parameter for server '%s'" % config.name) super(StreamingConnection, self).__init__(config, config.streaming_conninfo) # Make sure we connect using the 'replication' option which # triggers streaming replication protocol communication self.conn_parameters['replication'] = 'true' # Override 'dbname' parameter. This operation is required to mimic # the behaviour of pg_receivexlog and pg_basebackup self.conn_parameters['dbname'] = 'replication' # Rebuild the conninfo string from the modified parameter lists self.conninfo = self.encode_dsn(self.conn_parameters) def connect(self): """ Connect to the PostgreSQL server. It reuses an existing connection. :returns: the connection to the server """ if self._check_connection(): return self._conn # Build a connection and set autocommit self._conn = super(StreamingConnection, self).connect() self._conn.autocommit = True return self._conn def fetch_remote_status(self): """ Returns the status of the connection to the PostgreSQL server. This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ result = dict.fromkeys( ('connection_error', 'streaming_supported', 'streaming', 'systemid', 'timeline', 'xlogpos'), None) try: # If the server is too old to support `pg_receivexlog`, # exit immediately. # This needs to be protected by the try/except because # `self.server_version` can raise a PostgresConnectionError if self.server_version < 90200: result["streaming_supported"] = False return result result["streaming_supported"] = True # Execute a IDENTIFY_SYSYEM to check the connection cursor = self._cursor() cursor.execute("IDENTIFY_SYSTEM") row = cursor.fetchone() # If something has been returned, barman is connected # to a replication backend if row: result['streaming'] = True # IDENTIFY_SYSTEM always return at least two values result['systemid'] = row[0] result['timeline'] = row[1] # PostgreSQL 9.1+ returns also the current xlog flush location if len(row) > 2: result['xlogpos'] = row[2] except psycopg2.ProgrammingError: # This is not a streaming connection result['streaming'] = False except PostgresConnectionError as e: result['connection_error'] = str(e).strip() _logger.warn("Error retrieving PostgreSQL status: %s", str(e).strip()) return result def create_physical_repslot(self, slot_name): """ Create a physical replication slot using the streaming connection :param str slot_name: Replication slot name """ cursor = self._cursor() try: # In the following query, the slot name is directly passed # to the CREATE_REPLICATION_SLOT command, without any # quoting. This is a characteristic of the streaming # connection, otherwise if will fail with a generic # "syntax error" cursor.execute('CREATE_REPLICATION_SLOT %s PHYSICAL' % slot_name) except psycopg2.DatabaseError as exc: if exc.pgcode == DUPLICATE_OBJECT: # A replication slot with the same name exists raise PostgresDuplicateReplicationSlot() elif exc.pgcode == CONFIGURATION_LIMIT_EXCEEDED: # Unable to create a new physical replication slot. # All slots are full. raise PostgresReplicationSlotsFull() else: raise PostgresException(str(exc).strip()) def drop_repslot(self, slot_name): """ Drop a physical replication slot using the streaming connection :param str slot_name: Replication slot name """ cursor = self._cursor() try: # In the following query, the slot name is directly passed # to the DROP_REPLICATION_SLOT command, without any # quoting. This is a characteristic of the streaming # connection, otherwise if will fail with a generic # "syntax error" cursor.execute('DROP_REPLICATION_SLOT %s' % slot_name) except psycopg2.DatabaseError as exc: if exc.pgcode == UNDEFINED_OBJECT: # A replication slot with the that name does not exist raise PostgresInvalidReplicationSlot() if exc.pgcode == OBJECT_IN_USE: # The replication slot is still in use raise PostgresReplicationSlotInUse() else: raise PostgresException(str(exc).strip()) class PostgreSQLConnection(PostgreSQL): """ This class represents a standard client connection to a PostgreSQL server. """ # Streaming replication client types STANDBY = 1 WALSTREAMER = 2 ANY_STREAMING_CLIENT = (STANDBY, WALSTREAMER) def __init__(self, config): """ PostgreSQL connection constructor. :param barman.config.ServerConfig config: the server configuration """ # Check that 'conninfo' option is properly set if config.conninfo is None: raise ConninfoException( "Missing 'conninfo' parameter for server '%s'" % config.name) super(PostgreSQLConnection, self).__init__(config, config.conninfo) self.configuration_files = None def connect(self): """ Connect to the PostgreSQL server. It reuses an existing connection. """ if self._check_connection(): return self._conn self._conn = super(PostgreSQLConnection, self).connect() if (self._conn.server_version >= 90000 and 'application_name' not in self.conn_parameters): try: cur = self._conn.cursor() cur.execute('SET application_name TO barman') cur.close() # If psycopg2 fails to set the application name, # raise the appropriate exception except psycopg2.ProgrammingError as e: raise PostgresAppNameError(str(e).strip()) return self._conn @property def server_txt_version(self): """ Human readable version of PostgreSQL (returned by the server) """ try: cur = self._cursor() cur.execute("SELECT version()") return cur.fetchone()[0].split()[1] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving PostgreSQL version: %s", str(e).strip()) return None @property def has_pgespresso(self): """ Returns true if the `pgespresso` extension is available """ try: # pg_extension is only available from Postgres 9.1+ if self.server_version < 90100: return False cur = self._cursor() cur.execute("SELECT count(*) FROM pg_extension " "WHERE extname = 'pgespresso'") q_result = cur.fetchone()[0] return q_result > 0 except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving pgespresso information: %s", str(e).strip()) return None @property def is_in_recovery(self): """ Returns true if PostgreSQL server is in recovery mode (hot standby) """ try: # pg_is_in_recovery is only available from Postgres 9.0+ if self.server_version < 90000: return False cur = self._cursor() cur.execute("SELECT pg_is_in_recovery()") return cur.fetchone()[0] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error calling pg_is_in_recovery() function: %s", str(e).strip()) return None @property def is_superuser(self): """ Returns true if current user has superuser privileges """ try: cur = self._cursor() cur.execute('SELECT usesuper FROM pg_user ' 'WHERE usename = CURRENT_USER') return cur.fetchone()[0] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error calling is_superuser() function: %s", str(e).strip()) return None @property def current_xlog_info(self): """ Get detailed information about the current WAL position in PostgreSQL. This method returns a dictionary containing the following data: * location * file_name * file_offset * timestamp When executed on a standby server file_name and file_offset are always None :rtype: psycopg2.extras.DictRow """ try: cur = self._cursor(cursor_factory=DictCursor) if not self.is_in_recovery: cur.execute( "SELECT location, " "({pg_walfile_name_offset}(location)).*, " "CURRENT_TIMESTAMP AS timestamp " "FROM {pg_current_wal_lsn}() AS location" .format(**self.name_map)) return cur.fetchone() else: cur.execute( "SELECT location, " "NULL AS file_name, " "NULL AS file_offset, " "CURRENT_TIMESTAMP AS timestamp " "FROM {pg_last_wal_replay_lsn}() AS location" .format(**self.name_map)) return cur.fetchone() except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving current xlog " "detailed information: %s", str(e).strip()) return None @property def current_xlog_file_name(self): """ Get current WAL file from PostgreSQL :return str: current WAL file in PostgreSQL """ current_xlog_info = self.current_xlog_info if current_xlog_info is not None: return current_xlog_info['file_name'] return None @property def xlog_segment_size(self): """ Retrieve the size of one WAL file. In PostgreSQL 11, users will be able to change the WAL size at runtime. Up to PostgreSQL 10, included, the WAL size can be changed at compile time :return: The wal size (In bytes) """ # Prior to PostgreSQL 8.4, the wal segment size was not configurable, # even in compilation if self.server_version < 80400: return DEFAULT_XLOG_SEG_SIZE try: cur = self._cursor(cursor_factory=DictCursor) # We can't use the `get_setting` method here, because it # use `SHOW`, returning an human readable value such as "16MB", # while we prefer a raw value such as 16777216. cur.execute("SELECT setting " "FROM pg_settings " "WHERE name='wal_block_size'") result = cur.fetchone() wal_block_size = int(result[0]) cur.execute("SELECT setting " "FROM pg_settings " "WHERE name='wal_segment_size'") result = cur.fetchone() wal_segment_size = int(result[0]) return wal_block_size * wal_segment_size except ValueError as e: _logger.error("Error retrieving current xlog " "segment size: %s", str(e).strip()) return None @property def current_xlog_location(self): """ Get current WAL location from PostgreSQL :return str: current WAL location in PostgreSQL """ current_xlog_info = self.current_xlog_info if current_xlog_info is not None: return current_xlog_info['location'] return None @property def current_size(self): """ Returns the total size of the PostgreSQL server (requires superuser) """ if not self.is_superuser: return None try: cur = self._cursor() cur.execute( "SELECT sum(pg_tablespace_size(oid)) " "FROM pg_tablespace") return cur.fetchone()[0] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving PostgreSQL total size: %s", str(e).strip()) return None def get_archiver_stats(self): """ This method gathers statistics from pg_stat_archiver. Only for Postgres 9.4+ or greater. If not available, returns None. :return dict|None: a dictionary containing Postgres statistics from pg_stat_archiver or None """ try: # pg_stat_archiver is only available from Postgres 9.4+ if self.server_version < 90400: return None cur = self._cursor(cursor_factory=DictCursor) # Select from pg_stat_archiver statistics view, # retrieving statistics about WAL archiver process activity, # also evaluating if the server is archiving without issues # and the archived WALs per second rate. # # We are using current_settings to check for archive_mode=always. # current_setting does normalise its output so we can just # check for 'always' settings using a direct string # comparison cur.execute( "SELECT *, " "current_setting('archive_mode') IN ('on', 'always') " "AND (last_failed_wal IS NULL " "OR last_failed_wal LIKE '%.history' " "AND substring(last_failed_wal from 1 for 8) " "<= substring(last_archived_wal from 1 for 8) " "OR last_failed_time <= last_archived_time) " "AS is_archiving, " "CAST (archived_count AS NUMERIC) " "/ EXTRACT (EPOCH FROM age(now(), stats_reset)) " "AS current_archived_wals_per_second " "FROM pg_stat_archiver") return cur.fetchone() except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving pg_stat_archive data: %s", str(e).strip()) return None def fetch_remote_status(self): """ Get the status of the PostgreSQL server This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ # PostgreSQL settings to get from the server (requiring superuser) pg_superuser_settings = [ 'data_directory'] # PostgreSQL settings to get from the server pg_settings = [] pg_query_keys = [ 'server_txt_version', 'is_superuser', 'is_in_recovery', 'current_xlog', 'pgespresso_installed', 'replication_slot_support', 'replication_slot', 'synchronous_standby_names', ] # Initialise the result dictionary setting all the values to None result = dict.fromkeys(pg_superuser_settings + pg_settings + pg_query_keys, None) try: # check for wal_level only if the version is >= 9.0 if self.server_version >= 90000: pg_settings.append('wal_level') # retrieves superuser settings if self.is_superuser: for name in pg_superuser_settings: result[name] = self.get_setting(name) # retrieves standard settings for name in pg_settings: result[name] = self.get_setting(name) result['is_superuser'] = self.is_superuser result['is_in_recovery'] = self.is_in_recovery result['server_txt_version'] = self.server_txt_version result['pgespresso_installed'] = self.has_pgespresso result['current_xlog'] = self.current_xlog_file_name result['current_size'] = self.current_size result.update(self.get_configuration_files()) # Retrieve the replication_slot status result["replication_slot_support"] = False if self.server_version >= 90400: result["replication_slot_support"] = True if self.config.slot_name is not None: result["replication_slot"] = ( self.get_replication_slot(self.config.slot_name)) # Retrieve the list of synchronous standby names result["synchronous_standby_names"] = [] if self.server_version >= 90100: result["synchronous_standby_names"] = ( self.get_synchronous_standby_names()) except (PostgresConnectionError, psycopg2.Error) as e: _logger.warn("Error retrieving PostgreSQL status: %s", str(e).strip()) return result def get_setting(self, name): """ Get a Postgres setting with a given name :param name: a parameter name """ try: cur = self._cursor() cur.execute('SHOW "%s"' % name.replace('"', '""')) return cur.fetchone()[0] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving PostgreSQL setting '%s': %s", name.replace('"', '""'), str(e).strip()) return None def get_tablespaces(self): """ Returns a list of tablespaces or None if not present """ try: cur = self._cursor() if self.server_version >= 90200: cur.execute( "SELECT spcname, oid, " "pg_tablespace_location(oid) AS spclocation " "FROM pg_tablespace " "WHERE pg_tablespace_location(oid) != ''") else: cur.execute( "SELECT spcname, oid, spclocation " "FROM pg_tablespace WHERE spclocation != ''") # Generate a list of tablespace objects return [Tablespace._make(item) for item in cur.fetchall()] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving PostgreSQL tablespaces: %s", str(e).strip()) return None def get_configuration_files(self): """ Get postgres configuration files or an empty dictionary in case of error :rtype: dict """ if self.configuration_files: return self.configuration_files try: self.configuration_files = {} cur = self._cursor() cur.execute( "SELECT name, setting FROM pg_settings " "WHERE name IN ('config_file', 'hba_file', 'ident_file')") for cname, cpath in cur.fetchall(): self.configuration_files[cname] = cpath # Retrieve additional configuration files # If PostgresSQL is older than 8.4 disable this check if self.server_version >= 80400: cur.execute( "SELECT DISTINCT sourcefile AS included_file " "FROM pg_settings " "WHERE sourcefile IS NOT NULL " "AND sourcefile NOT IN " "(SELECT setting FROM pg_settings " "WHERE name = 'config_file') " "ORDER BY 1") # Extract the values from the containing single element tuples included_files = [included_file for included_file, in cur.fetchall()] if len(included_files) > 0: self.configuration_files['included_files'] = included_files except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving PostgreSQL configuration files " "location: %s", str(e).strip()) self.configuration_files = {} return self.configuration_files def create_restore_point(self, target_name): """ Create a restore point with the given target name The method executes the pg_create_restore_point() function through a PostgreSQL connection. Only for Postgres versions >= 9.1 when not in replication. If requirements are not met, the operation is skipped. :param str target_name: name of the restore point :returns: the restore point LSN :rtype: str|None """ if self.server_version < 90100: return None # Not possible if on a standby # Called inside the pg_connect context to reuse the connection if self.is_in_recovery: return None try: cur = self._cursor() cur.execute( "SELECT pg_create_restore_point(%s)", [target_name]) return cur.fetchone()[0] except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug('Error issuing pg_create_restore_point()' 'command: %s', str(e).strip()) return None def start_exclusive_backup(self, label): """ Calls pg_start_backup() on the PostgreSQL server This method returns a dictionary containing the following data: * location * file_name * file_offset * timestamp :param str label: descriptive string to identify the backup :rtype: psycopg2.extras.DictRow """ try: conn = self.connect() # Rollback to release the transaction, as the pg_start_backup # invocation can last up to PostgreSQL's checkpoint_timeout conn.rollback() # Start an exclusive backup cur = conn.cursor(cursor_factory=DictCursor) if self.server_version < 80400: cur.execute( "SELECT location, " "({pg_walfile_name_offset}(location)).*, " "now() AS timestamp " "FROM pg_start_backup(%s) AS location" .format(**self.name_map), (label,)) else: cur.execute( "SELECT location, " "({pg_walfile_name_offset}(location)).*, " "now() AS timestamp " "FROM pg_start_backup(%s,%s) AS location" .format(**self.name_map), (label, self.config.immediate_checkpoint)) start_row = cur.fetchone() # Rollback to release the transaction, as the connection # is to be retained until the end of backup conn.rollback() return start_row except (PostgresConnectionError, psycopg2.Error) as e: msg = "pg_start_backup(): %s" % str(e).strip() _logger.debug(msg) raise PostgresException(msg) def start_concurrent_backup(self, label): """ Calls pg_start_backup on the PostgreSQL server using the API introduced with version 9.6 This method returns a dictionary containing the following data: * location * timeline * timestamp :param str label: descriptive string to identify the backup :rtype: psycopg2.extras.DictRow """ try: conn = self.connect() # Rollback to release the transaction, as the pg_start_backup # invocation can last up to PostgreSQL's checkpoint_timeout conn.rollback() # Start the backup using the api introduced in postgres 9.6 cur = conn.cursor(cursor_factory=DictCursor) cur.execute( "SELECT location, " "(SELECT timeline_id " "FROM pg_control_checkpoint()) AS timeline, " "now() AS timestamp " "FROM pg_start_backup(%s, %s, FALSE) AS location", (label, self.config.immediate_checkpoint)) start_row = cur.fetchone() # Rollback to release the transaction, as the connection # is to be retained until the end of backup conn.rollback() return start_row except (PostgresConnectionError, psycopg2.Error) as e: msg = "pg_start_backup command: %s" % (str(e).strip(),) _logger.debug(msg) raise PostgresException(msg) def stop_exclusive_backup(self): """ Calls pg_stop_backup() on the PostgreSQL server This method returns a dictionary containing the following data: * location * file_name * file_offset * timestamp :rtype: psycopg2.extras.DictRow """ try: conn = self.connect() # Rollback to release the transaction, as the pg_stop_backup # invocation could will wait until the current WAL file is shipped conn.rollback() # Stop the backup cur = conn.cursor(cursor_factory=DictCursor) cur.execute( "SELECT location, " "({pg_walfile_name_offset}(location)).*, " "now() AS timestamp " "FROM pg_stop_backup() AS location" .format(**self.name_map) ) return cur.fetchone() except (PostgresConnectionError, psycopg2.Error) as e: msg = "Error issuing pg_stop_backup command: %s" % str(e).strip() _logger.debug(msg) raise PostgresException( 'Cannot terminate exclusive backup. ' 'You might have to manually execute pg_stop_backup ' 'on your PostgreSQL server') def stop_concurrent_backup(self): """ Calls pg_stop_backup on the PostgreSQL server using the API introduced with version 9.6 This method returns a dictionary containing the following data: * location * timeline * backup_label * timestamp :rtype: psycopg2.extras.DictRow """ try: conn = self.connect() # Rollback to release the transaction, as the pg_stop_backup # invocation could will wait until the current WAL file is shipped conn.rollback() # Stop the backup using the api introduced with version 9.6 cur = conn.cursor(cursor_factory=DictCursor) cur.execute( 'SELECT end_row.lsn AS location, ' '(SELECT CASE WHEN pg_is_in_recovery() ' 'THEN min_recovery_end_timeline ELSE timeline_id END ' 'FROM pg_control_checkpoint(), pg_control_recovery()' ') AS timeline, ' 'end_row.labelfile AS backup_label, ' 'now() AS timestamp FROM pg_stop_backup(FALSE) AS end_row') return cur.fetchone() except (PostgresConnectionError, psycopg2.Error) as e: msg = "Error issuing pg_stop_backup command: %s" % str(e).strip() _logger.debug(msg) raise PostgresException(msg) def pgespresso_start_backup(self, label): """ Execute a pgespresso_start_backup This method returns a dictionary containing the following data: * backup_label * timestamp :param str label: descriptive string to identify the backup :rtype: psycopg2.extras.DictRow """ try: conn = self.connect() # Rollback to release the transaction, # as the pgespresso_start_backup invocation can last # up to PostgreSQL's checkpoint_timeout conn.rollback() # Start the concurrent backup using pgespresso cur = conn.cursor(cursor_factory=DictCursor) cur.execute( 'SELECT pgespresso_start_backup(%s,%s) AS backup_label, ' 'now() AS timestamp', (label, self.config.immediate_checkpoint)) start_row = cur.fetchone() # Rollback to release the transaction, as the connection # is to be retained until the end of backup conn.rollback() return start_row except (PostgresConnectionError, psycopg2.Error) as e: msg = "pgespresso_start_backup(): %s" % str(e).strip() _logger.debug(msg) raise PostgresException(msg) def pgespresso_stop_backup(self, backup_label): """ Execute a pgespresso_stop_backup This method returns a dictionary containing the following data: * end_wal * timestamp :param str backup_label: backup label as returned by pgespress_start_backup :rtype: psycopg2.extras.DictRow """ try: conn = self.connect() # Issue a rollback to release any unneeded lock conn.rollback() cur = conn.cursor(cursor_factory=DictCursor) cur.execute("SELECT pgespresso_stop_backup(%s) AS end_wal, " "now() AS timestamp", (backup_label,)) return cur.fetchone() except (PostgresConnectionError, psycopg2.Error) as e: msg = "Error issuing pgespresso_stop_backup() command: %s" % ( str(e).strip()) _logger.debug(msg) raise PostgresException( '%s\n' 'HINT: You might have to manually execute ' 'pgespresso_abort_backup() on your PostgreSQL ' 'server' % msg) def switch_wal(self): """ Execute a pg_switch_wal() To be SURE of the switch of a xlog, we collect the xlogfile name before and after the switch. The method returns the just closed xlog file name if the current xlog file has changed, it returns an empty string otherwise. The method returns None if something went wrong during the execution of the pg_switch_wal command. :rtype: str|None """ try: conn = self.connect() # Requires superuser privilege if not self.is_superuser: raise PostgresSuperuserRequired() # If this server is in recovery there is nothing to do if self.is_in_recovery: raise PostgresIsInRecovery() cur = conn.cursor() # Collect the xlog file name before the switch cur.execute('SELECT {pg_walfile_name}(' '{pg_current_wal_insert_lsn}())' .format(**self.name_map)) pre_switch = cur.fetchone()[0] # Switch cur.execute('SELECT {pg_walfile_name}({pg_switch_wal}())' .format(**self.name_map)) # Collect the xlog file name after the switch cur.execute('SELECT {pg_walfile_name}(' '{pg_current_wal_insert_lsn}())' .format(**self.name_map)) post_switch = cur.fetchone()[0] if pre_switch < post_switch: return pre_switch else: return '' except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug( "Error issuing {pg_switch_wal}() command: %s" .format(**self.name_map), str(e).strip()) return None def checkpoint(self): """ Execute a checkpoint """ try: conn = self.connect() # Requires superuser privilege if not self.is_superuser: raise PostgresSuperuserRequired() cur = conn.cursor() cur.execute("CHECKPOINT") except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug( "Error issuing CHECKPOINT: %s", str(e).strip()) def get_replication_stats(self, client_type=STANDBY): """ Returns streaming replication information """ try: cur = self._cursor(cursor_factory=NamedTupleCursor) # Without superuser rights, this function is useless # TODO: provide a simplified version for non-superusers if not self.is_superuser: raise PostgresSuperuserRequired() # pg_stat_replication is a system view that contains one # row per WAL sender process with information about the # replication status of a standby server. It has been # introduced in PostgreSQL 9.1. Current fields are: # # - pid (procpid in 9.1) # - usesysid # - usename # - application_name # - client_addr # - client_hostname # - client_port # - backend_start # - backend_xmin (9.4+) # - state # - sent_lsn (sent_location before 10) # - write_lsn (write_location before 10) # - flush_lsn (flush_location before 10) # - replay_lsn (replay_location before 10) # - sync_priority # - sync_state # if self.server_version < 90100: raise PostgresUnsupportedFeature('9.1') from_repslot = "" if self.server_version >= 100000: # Current implementation (10+) what = "r.*, rs.slot_name" # Look for replication slot name from_repslot = "LEFT JOIN pg_replication_slots rs " \ "ON (r.pid = rs.active_pid) " elif self.server_version >= 90500: # PostgreSQL 9.5/9.6 what = "pid, " \ "usesysid, " \ "usename, " \ "application_name, " \ "client_addr, " \ "client_hostname, " \ "client_port, " \ "backend_start, " \ "backend_xmin, " \ "state, " \ "sent_location AS sent_lsn, " \ "write_location AS write_lsn, " \ "flush_location AS flush_lsn, " \ "replay_location AS replay_lsn, " \ "sync_priority, " \ "sync_state, " \ "rs.slot_name" # Look for replication slot name from_repslot = "LEFT JOIN pg_replication_slots rs " \ "ON (r.pid = rs.active_pid) " elif self.server_version >= 90400: # PostgreSQL 9.4 what = "pid, " \ "usesysid, " \ "usename, " \ "application_name, " \ "client_addr, " \ "client_hostname, " \ "client_port, " \ "backend_start, " \ "backend_xmin, " \ "state, " \ "sent_location AS sent_lsn, " \ "write_location AS write_lsn, " \ "flush_location AS flush_lsn, " \ "replay_location AS replay_lsn, " \ "sync_priority, " \ "sync_state" elif self.server_version >= 90200: # PostgreSQL 9.2/9.3 what = "pid, " \ "usesysid, " \ "usename, " \ "application_name, " \ "client_addr, " \ "client_hostname, " \ "client_port, " \ "backend_start, " \ "CAST (NULL AS xid) AS backend_xmin, " \ "state, " \ "sent_location AS sent_lsn, " \ "write_location AS write_lsn, " \ "flush_location AS flush_lsn, " \ "replay_location AS replay_lsn, " \ "sync_priority, " \ "sync_state" else: # PostgreSQL 9.1 what = "procpid AS pid, " \ "usesysid, " \ "usename, " \ "application_name, " \ "client_addr, " \ "client_hostname, " \ "client_port, " \ "backend_start, " \ "CAST (NULL AS xid) AS backend_xmin, " \ "state, " \ "sent_location AS sent_lsn, " \ "write_location AS write_lsn, " \ "flush_location AS flush_lsn, " \ "replay_location AS replay_lsn, " \ "sync_priority, " \ "sync_state" # Streaming client if client_type == self.STANDBY: # Standby server where = 'WHERE {replay_lsn} IS NOT NULL '.format( **self.name_map) elif client_type == self.WALSTREAMER: # WAL streamer where = 'WHERE {replay_lsn} IS NULL '.format( **self.name_map) else: where = '' # Execute the query cur.execute( "SELECT %s, " "pg_is_in_recovery() AS is_in_recovery, " "CASE WHEN pg_is_in_recovery() " " THEN {pg_last_wal_receive_lsn}() " " ELSE {pg_current_wal_lsn}() " "END AS current_lsn " "FROM pg_stat_replication r " "%s" "%s" "ORDER BY sync_state DESC, sync_priority" .format(**self.name_map) % (what, from_repslot, where)) # Generate a list of standby objects return cur.fetchall() except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving status of standby servers: %s", str(e).strip()) return None def get_replication_slot(self, slot_name): """ Retrieve from the PostgreSQL server a physical replication slot with a specific slot_name. This method returns a dictionary containing the following data: * slot_name * active * restart_lsn :param str slot_name: the replication slot name :rtype: psycopg2.extras.DictRow """ if self.server_version < 90400: # Raise exception if replication slot are not supported # by PostgreSQL version raise PostgresUnsupportedFeature('9.4') else: cur = self._cursor(cursor_factory=NamedTupleCursor) try: cur.execute("SELECT slot_name, " "active, " "restart_lsn " "FROM pg_replication_slots " "WHERE slot_type = 'physical' " "AND slot_name = '%s'" % slot_name) # Retrieve the replication slot information return cur.fetchone() except (PostgresConnectionError, psycopg2.Error) as e: _logger.debug("Error retrieving replication_slots: %s", str(e).strip()) raise def get_synchronous_standby_names(self): """ Retrieve the list of named synchronous standby servers from PostgreSQL This method returns a list of names :return list: synchronous standby names """ if self.server_version < 90100: # Raise exception if synchronous replication is not supported raise PostgresUnsupportedFeature('9.1') else: synchronous_standby_names = ( self.get_setting('synchronous_standby_names')) # Normalise the list of sync standby names # On PostgreSQL 9.6 it is possible to specify the number of # required synchronous standby using this format: # n (name1, name2, ... nameN). # We only need the name list, so we discard everything else. # The name list starts after the first parenthesis or at pos 0 names_start = synchronous_standby_names.find('(') + 1 names_end = synchronous_standby_names.rfind(')') if names_end < 0: names_end = len(synchronous_standby_names) names_list = synchronous_standby_names[names_start:names_end] return [x.strip() for x in names_list.split(',')] @property def name_map(self): """ Return a map with function and directory names according to the current PostgreSQL version. Each entry has the `current` name as key and the name for the specific version as value. :rtype: dict[str] """ # Avoid raising an error if the connection is not available try: server_version = self.server_version except PostgresConnectionError: _logger.debug('Impossible to detect the PostgreSQL version, ' 'name_map will return names from latest version') server_version = None if server_version and server_version < 100000: return { 'pg_switch_wal': 'pg_switch_xlog', 'pg_walfile_name': 'pg_xlogfile_name', 'pg_wal': 'pg_xlog', 'pg_walfile_name_offset': 'pg_xlogfile_name_offset', 'pg_last_wal_replay_lsn': 'pg_last_xlog_replay_location', 'pg_current_wal_lsn': 'pg_current_xlog_location', 'pg_current_wal_insert_lsn': 'pg_current_xlog_insert_location', 'pg_last_wal_receive_lsn': 'pg_last_xlog_receive_location', 'sent_lsn': 'sent_location', 'write_lsn': 'write_location', 'flush_lsn': 'flush_location', 'replay_lsn': 'replay_location', } return { 'pg_switch_wal': 'pg_switch_wal', 'pg_walfile_name': 'pg_walfile_name', 'pg_wal': 'pg_wal', 'pg_walfile_name_offset': 'pg_walfile_name_offset', 'pg_last_wal_replay_lsn': 'pg_last_wal_replay_lsn', 'pg_current_wal_lsn': 'pg_current_wal_lsn', 'pg_current_wal_insert_lsn': 'pg_current_wal_insert_lsn', 'pg_last_wal_receive_lsn': 'pg_last_wal_receive_lsn', 'sent_lsn': 'sent_lsn', 'write_lsn': 'write_lsn', 'flush_lsn': 'flush_lsn', 'replay_lsn': 'replay_lsn', } barman-2.3/barman/process.py0000644000076500000240000001333213134472625016654 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see import errno import logging import os import signal import time from glob import glob from barman import output from barman.exceptions import LockFileParsingError from barman.lockfile import ServerWalReceiveLock _logger = logging.getLogger(__name__) class ProcessInfo(object): """ Barman process representation """ def __init__(self, pid, server_name, task): """ This object contains all the information required to identify a barman process :param int pid: Process ID :param string server_name: Name of the server owning the process :param string task: Task name (receive-wal, archive-wal...) """ self.pid = pid self.server_name = server_name self.task = task class ProcessManager(object): """ Class for the management of barman processes owned by a server """ # Map containing the tasks we want to retrieve (and eventually manage) TASKS = { 'receive-wal': ServerWalReceiveLock } def __init__(self, config): """ Build a ProcessManager for the provided server :param config: configuration of the server owning the process manager """ self.config = config self.process_list = [] # Cycle over the lock files in the lock directory for this server for path in glob(os.path.join(self.config.barman_lock_directory, '.%s-*.lock' % self.config.name)): for task, lock_class in self.TASKS.items(): # Check the lock_name against the lock class lock = lock_class.build_if_matches(path) if lock: try: # Use the lock to get the owner pid pid = lock.get_owner_pid() except LockFileParsingError: _logger.warning( "Skipping the %s process for server %s: " "Error reading the PID from lock file '%s'", task, self.config.name, path) break # If there is a pid save it in the process list if pid: self.process_list.append( ProcessInfo(pid, config.name, task)) # In any case, we found a match, so we must stop iterating # over the task types and handle the the next path break def list(self, task_filter=None): """ Returns a list of processes owned by this server If no filter is provided, all the processes are returned. :param str task_filter: Type of process we want to retrieve :return list[ProcessInfo]: List of processes for the server """ server_tasks = [] for process in self.process_list: # Filter the processes if necessary if task_filter and process.task != task_filter: continue server_tasks.append(process) return server_tasks def kill(self, process_info, retries=10): """ Kill a process Returns True if killed successfully False otherwise :param ProcessInfo process_info: representation of the process we want to kill :param int retries: number of times the method will check if the process is still alive :rtype: bool """ # Try to kill the process try: _logger.debug("Sending SIGINT to PID %s", process_info.pid) os.kill(process_info.pid, signal.SIGINT) _logger.debug("os.kill call succeeded") except OSError as e: _logger.debug("os.kill call failed: %s", e) # The process doesn't exists. It has probably just terminated. if e.errno == errno.ESRCH: return True # Something unexpected has happened output.error("%s", e) return False # Check if the process have been killed. the fastest (and maybe safest) # way is to send a kill with 0 as signal. # If the method returns an OSError exceptions, the process have been # killed successfully, otherwise is still alive. for counter in range(retries): try: _logger.debug("Checking with SIG_DFL if PID %s is still alive", process_info.pid) os.kill(process_info.pid, signal.SIG_DFL) _logger.debug("os.kill call succeeded") except OSError as e: _logger.debug("os.kill call failed: %s", e) # If the process doesn't exists, we are done. if e.errno == errno.ESRCH: return True # Something unexpected has happened output.error("%s", e) return False time.sleep(1) _logger.debug("The PID %s has not been terminated after %s retries", process_info.pid, retries) return False barman-2.3/barman/recovery_executor.py0000644000076500000240000013202113151271540020737 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module contains the methods necessary to perform a recovery """ from __future__ import print_function import collections import logging import os import re import shutil import socket import tempfile import time from io import StringIO import dateutil.parser import dateutil.tz from barman import output, xlog from barman.command_wrappers import RsyncPgData from barman.config import RecoveryOptions from barman.copy_controller import RsyncCopyController from barman.exceptions import (BadXlogSegmentName, CommandFailedException, DataTransferFailure, FsOperationFailed) from barman.fs import UnixLocalCommand, UnixRemoteCommand from barman.infofile import BackupInfo from barman.utils import mkpath # generic logger for this module _logger = logging.getLogger(__name__) # regexp matching a single value in Postgres configuration file PG_CONF_SETTING_RE = re.compile(r"^\s*([^\s=]+)\s*=?\s*(.*)$") # create a namedtuple object called Assertion # with 'filename', 'line', 'key' and 'value' as properties Assertion = collections.namedtuple('Assertion', 'filename line key value') # noinspection PyMethodMayBeStatic class RecoveryExecutor(object): """ Class responsible of recovery operations """ # Potentially dangerous options list, which need to be revised by the user # after a recovery DANGEROUS_OPTIONS = ['data_directory', 'config_file', 'hba_file', 'ident_file', 'external_pid_file', 'ssl_cert_file', 'ssl_key_file', 'ssl_ca_file', 'ssl_crl_file', 'unix_socket_directory', 'include', 'include_dir', 'include_if_exists'] # List of options that, if present, need to be forced to a specific value # during recovery, to avoid data losses MANGLE_OPTIONS = {'archive_command': 'false'} def __init__(self, backup_manager): """ Constructor :param barman.backup.BackupManager backup_manager: the BackupManager owner of the executor """ self.backup_manager = backup_manager self.server = backup_manager.server self.config = backup_manager.config def recover(self, backup_info, dest, tablespaces, target_tli, target_time, target_xid, target_name, target_immediate, exclusive, remote_command): """ Performs a recovery of a backup :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None target_tli: the target timeline :param str|None target_time: the target time :param str|None target_xid: the target xid :param str|None target_name: the target name created previously with pg_create_restore_point() function call :param str|None target_immediate: end recovery as soon as consistency is reached :param bool exclusive: whether the recovery is exclusive or not :param str|None remote_command: The remote command to recover the base backup, in case of remote backup. """ # Run the cron to be sure the wal catalog is up to date # Prepare a map that contains all the objects required for a recovery recovery_info = self._setup(backup_info, remote_command, dest) output.info("Starting %s restore for server %s using backup %s", recovery_info['recovery_dest'], self.server.config.name, backup_info.backup_id) output.info("Destination directory: %s", dest) # Set targets for PITR self._set_pitr_targets(recovery_info, backup_info, dest, target_name, target_time, target_tli, target_xid, target_immediate) # Retrieve the safe_horizon for smart copy self._retrieve_safe_horizon(recovery_info, backup_info, dest) # check destination directory. If doesn't exist create it try: recovery_info['cmd'].create_dir_if_not_exists(dest) except FsOperationFailed as e: output.error("unable to initialise destination directory " "'%s': %s", dest, e) output.close_and_exit() # Initialize tablespace directories if backup_info.tablespaces: self._prepare_tablespaces(backup_info, recovery_info['cmd'], dest, tablespaces) # Copy the base backup output.info("Copying the base backup.") try: self._backup_copy( backup_info, dest, tablespaces, remote_command, recovery_info['safe_horizon']) except DataTransferFailure as e: output.error("Failure copying base backup: %s", e) output.close_and_exit() # Copy the backup.info file in the destination as # ".barman-recover.info" if remote_command: try: recovery_info['rsync'](backup_info.filename, ':%s/.barman-recover.info' % dest) except CommandFailedException as e: output.error( 'copy of recovery metadata file failed: %s', e) output.close_and_exit() else: backup_info.save(os.path.join(dest, '.barman-recover.info')) # Restore the WAL segments. If GET_WAL option is set, skip this phase # as they will be retrieved using the wal-get command. if not recovery_info['get_wal']: output.info("Copying required WAL segments.") try: # Retrieve a list of required log files required_xlog_files = tuple( self.server.get_required_xlog_files( backup_info, target_tli, recovery_info['target_epoch'])) # Restore WAL segments into the wal_dest directory self._xlog_copy(required_xlog_files, recovery_info['wal_dest'], remote_command) except DataTransferFailure as e: output.error("Failure copying WAL files: %s", e) output.close_and_exit() except BadXlogSegmentName as e: output.error( "invalid xlog segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", str(e), self.config.name) output.close_and_exit() # If WAL files are put directly in the pg_xlog directory, # avoid shipping of just recovered files # by creating the corresponding archive status file if not recovery_info['is_pitr']: output.info("Generating archive status files") self._generate_archive_status(recovery_info, remote_command, required_xlog_files) # Generate recovery.conf file (only if needed by PITR) if recovery_info['is_pitr']: output.info("Generating recovery.conf") self._generate_recovery_conf(recovery_info, backup_info, dest, target_immediate, exclusive, remote_command, target_name, target_time, target_tli, target_xid) # Create archive_status directory if necessary archive_status_dir = os.path.join(recovery_info['wal_dest'], 'archive_status') try: recovery_info['cmd'].create_dir_if_not_exists(archive_status_dir) except FsOperationFailed as e: output.error("unable to create the archive_status directory " "'%s': %s", archive_status_dir, e) output.close_and_exit() # As last step, analyse configuration files in order to spot # harmful options. Barman performs automatic conversion of # some options as well as notifying users of their existence. # # This operation is performed in three steps: # 1) mapping # 2) analysis # 3) copy output.info("Identify dangerous settings in destination directory.") self._map_temporary_config_files(recovery_info, backup_info, remote_command) self._analyse_temporary_config_files(recovery_info) self._copy_temporary_config_files(dest, remote_command, recovery_info) # Cleanup operations self._teardown(recovery_info) return recovery_info def _setup(self, backup_info, remote_command, dest): """ Prepare the recovery_info dictionary for the recovery, as well as temporary working directory :param barman.infofile.BackupInfo backup_info: representation of a backup :param str remote_command: ssh command for remote connection :return dict: recovery_info dictionary, holding the basic values for a recovery """ # Calculate the name of the WAL directory if backup_info.version < 100000: wal_dest = os.path.join(dest, 'pg_xlog') else: wal_dest = os.path.join(dest, 'pg_wal') recovery_info = { 'cmd': None, 'recovery_dest': 'local', 'rsync': None, 'configuration_files': [], 'destination_path': dest, 'temporary_configuration_files': [], 'tempdir': tempfile.mkdtemp(prefix='barman_recovery-'), 'is_pitr': False, 'wal_dest': wal_dest, 'get_wal': RecoveryOptions.GET_WAL in self.config.recovery_options, } # A map that will keep track of the results of the recovery. # Used for output generation results = { 'changes': [], 'warnings': [], 'delete_barman_xlog': False, 'missing_files': [], 'get_wal': False, } recovery_info['results'] = results # Set up a list of configuration files recovery_info['configuration_files'].append('postgresql.conf') if backup_info.version >= 90400: recovery_info['configuration_files'].append('postgresql.auto.conf') # Handle remote recovery options if remote_command: recovery_info['recovery_dest'] = 'remote' try: recovery_info['rsync'] = RsyncPgData( path=self.server.path, ssh=remote_command, bwlimit=self.config.bandwidth_limit, network_compression=self.config.network_compression) except CommandFailedException: self._teardown(recovery_info) raise try: # create a UnixRemoteCommand obj if is a remote recovery recovery_info['cmd'] = UnixRemoteCommand(remote_command, path=self.server.path) except FsOperationFailed: self._teardown(recovery_info) output.error( "Unable to connect to the target host using the command " "'%s'", remote_command) output.close_and_exit() else: # if is a local recovery create a UnixLocalCommand recovery_info['cmd'] = UnixLocalCommand() return recovery_info def _set_pitr_targets(self, recovery_info, backup_info, dest, target_name, target_time, target_tli, target_xid, target_immediate): """ Set PITR targets - as specified by the user :param dict recovery_info: Dictionary containing all the recovery parameters :param barman.infofile.BackupInfo backup_info: representation of a backup :param str dest: destination directory of the recovery :param str|None target_name: recovery target name for PITR :param str|None target_time: recovery target time for PITR :param str|None target_tli: recovery target timeline for PITR :param str|None target_xid: recovery target transaction id for PITR :param bool|None target_immediate: end recovery as soon as consistency is reached """ target_epoch = None target_datetime = None # Detect PITR if (target_time or target_xid or (target_tli and target_tli != backup_info.timeline) or target_name or recovery_info['get_wal'] or (backup_info.version >= 90400 and target_immediate)): recovery_info['is_pitr'] = True targets = {} if target_time: # noinspection PyBroadException try: target_datetime = dateutil.parser.parse(target_time) except ValueError as e: output.error( "unable to parse the target time parameter %r: %s", target_time, e) self._teardown(recovery_info) output.close_and_exit() except Exception: # this should not happen, but there is a known bug in # dateutil.parser.parse() implementation # ref: https://bugs.launchpad.net/dateutil/+bug/1247643 output.error( "unable to parse the target time parameter %r", target_time) output.close_and_exit() target_epoch = ( time.mktime(target_datetime.timetuple()) + (target_datetime.microsecond / 1000000.)) targets['time'] = str(target_datetime) if target_xid: targets['xid'] = str(target_xid) if target_tli and target_tli != backup_info.timeline: targets['timeline'] = str(target_tli) if target_name: targets['name'] = str(target_name) if target_immediate: targets['immediate'] = target_immediate output.info( "Doing PITR. Recovery target %s", (", ".join(["%s: %r" % (k, v) for k, v in targets.items()]))) recovery_info['wal_dest'] = os.path.join(dest, 'barman_xlog') # With a PostgreSQL version older than 8.4, it is the user's # responsibility to delete the "barman_xlog" directory as the # restore_command option in recovery.conf is not supported if backup_info.version < 80400 and \ not recovery_info['get_wal']: recovery_info['results']['delete_barman_xlog'] = True recovery_info['target_epoch'] = target_epoch recovery_info['target_datetime'] = target_datetime def _retrieve_safe_horizon(self, recovery_info, backup_info, dest): """ Retrieve the safe_horizon for smart copy If the target directory contains a previous recovery, it is safe to pick the least of the two backup "begin times" (the one we are recovering now and the one previously recovered in the target directory). Set the value in the given recovery_info dictionary. :param dict recovery_info: Dictionary containing all the recovery parameters :param barman.infofile.BackupInfo backup_info: a backup representation :param str dest: recovery destination directory """ # noinspection PyBroadException try: backup_begin_time = backup_info.begin_time # Retrieve previously recovered backup metadata (if available) dest_info_txt = recovery_info['cmd'].get_file_content( os.path.join(dest, '.barman-recover.info')) dest_info = BackupInfo( self.server, info_file=StringIO(dest_info_txt)) dest_begin_time = dest_info.begin_time # Pick the earlier begin time. Both are tz-aware timestamps because # BackupInfo class ensure it safe_horizon = min(backup_begin_time, dest_begin_time) output.info("Using safe horizon time for smart rsync copy: %s", safe_horizon) except FsOperationFailed as e: # Setting safe_horizon to None will effectively disable # the time-based part of smart_copy method. However it is still # faster than running all the transfers with checksum enabled. # # FsOperationFailed means the .barman-recover.info is not available # on destination directory safe_horizon = None _logger.warning('Unable to retrieve safe horizon time ' 'for smart rsync copy: %s', e) except Exception as e: # Same as above, but something failed decoding .barman-recover.info # or comparing times, so log the full traceback safe_horizon = None _logger.exception('Error retrieving safe horizon time ' 'for smart rsync copy: %s', e) recovery_info['safe_horizon'] = safe_horizon def _prepare_tablespaces(self, backup_info, cmd, dest, tablespaces): """ Prepare the directory structure for required tablespaces, taking care of tablespaces relocation, if requested. :param barman.infofile.BackupInfo backup_info: backup representation :param barman.fs.UnixLocalCommand cmd: Object for filesystem interaction :param str dest: destination dir for the recovery :param dict tablespaces: dict of all the tablespaces and their location """ tblspc_dir = os.path.join(dest, 'pg_tblspc') try: # check for pg_tblspc dir into recovery destination folder. # if it does not exists, create it cmd.create_dir_if_not_exists(tblspc_dir) except FsOperationFailed as e: output.error("unable to initialise tablespace directory " "'%s': %s", tblspc_dir, e) output.close_and_exit() for item in backup_info.tablespaces: # build the filename of the link under pg_tblspc directory pg_tblspc_file = os.path.join(tblspc_dir, str(item.oid)) # by default a tablespace goes in the same location where # it was on the source server when the backup was taken location = item.location # if a relocation has been requested for this tablespace, # use the target directory provided by the user if tablespaces and item.name in tablespaces: location = tablespaces[item.name] try: # remove the current link in pg_tblspc, if it exists cmd.delete_if_exists(pg_tblspc_file) # create tablespace location, if does not exist # (raise an exception if it is not possible) cmd.create_dir_if_not_exists(location) # check for write permissions on destination directory cmd.check_write_permission(location) # create symlink between tablespace and recovery folder cmd.create_symbolic_link(location, pg_tblspc_file) except FsOperationFailed as e: output.error("unable to prepare '%s' tablespace " "(destination '%s'): %s", item.name, location, e) output.close_and_exit() output.info("\t%s, %s, %s", item.oid, item.name, location) def _backup_copy(self, backup_info, dest, tablespaces=None, remote_command=None, safe_horizon=None): """ Perform the actual copy of the base backup for recovery purposes First, it copies one tablespace at a time, then the PGDATA directory. Bandwidth limitation, according to configuration, is applied in the process. TODO: manage configuration files if outside PGDATA. :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. :param datetime.datetime|None safe_horizon: anything after this time has to be checked with checksum """ # Set a ':' prefix to remote destinations dest_prefix = '' if remote_command: dest_prefix = ':' # Create the copy controller object, specific for rsync, # which will drive all the copy operations. Items to be # copied are added before executing the copy() method controller = RsyncCopyController( path=self.server.path, ssh_command=remote_command, network_compression=self.config.network_compression, safe_horizon=safe_horizon, retry_times=self.config.basebackup_retry_times, retry_sleep=self.config.basebackup_retry_sleep, workers=self.config.parallel_jobs, ) # Dictionary for paths to be excluded from rsync exclude_and_protect = [] # Process every tablespace if backup_info.tablespaces: for tablespace in backup_info.tablespaces: # By default a tablespace goes in the same location where # it was on the source server when the backup was taken location = tablespace.location # If a relocation has been requested for this tablespace # use the user provided target directory if tablespaces and tablespace.name in tablespaces: location = tablespaces[tablespace.name] # If the tablespace location is inside the data directory, # exclude and protect it from being deleted during # the data directory copy if location.startswith(dest): exclude_and_protect += [location[len(dest):]] # Exclude and protect the tablespace from being deleted during # the data directory copy exclude_and_protect.append("/pg_tblspc/%s" % tablespace.oid) # Add the tablespace directory to the list of objects # to be copied by the controller controller.add_directory( label=tablespace.name, src='%s/' % backup_info.get_data_directory(tablespace.oid), dst=dest_prefix + location, bwlimit=self.config.get_bwlimit(tablespace), item_class=controller.TABLESPACE_CLASS ) # Add the PGDATA directory to the list of objects to be copied # by the controller controller.add_directory( label='pgdata', src='%s/' % backup_info.get_data_directory(), dst=dest_prefix + dest, bwlimit=self.config.get_bwlimit(), exclude=[ '/pg_log/*', '/pg_xlog/*', '/pg_wal/*', '/postmaster.pid', '/recovery.conf', '/tablespace_map', ], exclude_and_protect=exclude_and_protect, item_class=controller.PGDATA_CLASS ) # TODO: Manage different location for configuration files # TODO: that were not within the data directory # Execute the copy try: controller.copy() # TODO: Improve the exception output except CommandFailedException as e: msg = "data transfer failure" raise DataTransferFailure.from_command_error( 'rsync', e, msg) def _xlog_copy(self, required_xlog_files, wal_dest, remote_command): """ Restore WAL segments :param required_xlog_files: list of all required WAL files :param wal_dest: the destination directory for xlog recover :param remote_command: default None. The remote command to recover the xlog, in case of remote backup. """ # List of required WAL files partitioned by containing directory xlogs = collections.defaultdict(list) # add '/' suffix to ensure it is a directory wal_dest = '%s/' % wal_dest # Map of every compressor used with any WAL file in the archive, # to be used during this recovery compressors = {} compression_manager = self.backup_manager.compression_manager # Fill xlogs and compressors maps from required_xlog_files for wal_info in required_xlog_files: hashdir = xlog.hash_dir(wal_info.name) xlogs[hashdir].append(wal_info) # If a compressor is required, make sure it exists in the cache if wal_info.compression is not None and \ wal_info.compression not in compressors: compressors[wal_info.compression] = \ compression_manager.get_compressor( compression=wal_info.compression) rsync = RsyncPgData( path=self.server.path, ssh=remote_command, bwlimit=self.config.bandwidth_limit, network_compression=self.config.network_compression) # If compression is used and this is a remote recovery, we need a # temporary directory where to spool uncompressed files, # otherwise we either decompress every WAL file in the local # destination, or we ship the uncompressed file remotely if compressors: if remote_command: # Decompress to a temporary spool directory wal_decompression_dest = tempfile.mkdtemp( prefix='barman_xlog-') else: # Decompress directly to the destination directory wal_decompression_dest = wal_dest # Make sure wal_decompression_dest exists mkpath(wal_decompression_dest) else: # If no compression wal_decompression_dest = None if remote_command: # If remote recovery tell rsync to copy them remotely # add ':' prefix to mark it as remote wal_dest = ':%s' % wal_dest total_wals = sum(map(len, xlogs.values())) partial_count = 0 for prefix in sorted(xlogs): batch_len = len(xlogs[prefix]) partial_count += batch_len source_dir = os.path.join(self.config.wals_directory, prefix) _logger.info( "Starting copy of %s WAL files %s/%s from %s to %s", batch_len, partial_count, total_wals, xlogs[prefix][0], xlogs[prefix][-1]) # If at least one compressed file has been found, activate # compression check and decompression for each WAL files if compressors: for segment in xlogs[prefix]: dst_file = os.path.join(wal_decompression_dest, segment.name) if segment.compression is not None: compressors[segment.compression].decompress( os.path.join(source_dir, segment.name), dst_file) else: shutil.copy2(os.path.join(source_dir, segment.name), dst_file) if remote_command: try: # Transfer the WAL files rsync.from_file_list( list(segment.name for segment in xlogs[prefix]), wal_decompression_dest, wal_dest) except CommandFailedException as e: msg = ("data transfer failure while copying WAL files " "to directory '%s'") % (wal_dest[1:],) raise DataTransferFailure.from_command_error( 'rsync', e, msg) # Cleanup files after the transfer for segment in xlogs[prefix]: file_name = os.path.join(wal_decompression_dest, segment.name) try: os.unlink(file_name) except OSError as e: output.warning( "Error removing temporary file '%s': %s", file_name, e) else: try: rsync.from_file_list( list(segment.name for segment in xlogs[prefix]), "%s/" % os.path.join(self.config.wals_directory, prefix), wal_dest) except CommandFailedException as e: msg = "data transfer failure while copying WAL files " \ "to directory '%s'" % (wal_dest[1:],) raise DataTransferFailure.from_command_error( 'rsync', e, msg) _logger.info("Finished copying %s WAL files.", total_wals) # Remove local decompression target directory if different from the # destination directory (it happens when compression is in use during a # remote recovery if wal_decompression_dest and wal_decompression_dest != wal_dest: shutil.rmtree(wal_decompression_dest) def _generate_archive_status(self, recovery_info, remote_command, required_xlog_files): """ Populate the archive_status directory :param dict recovery_info: Dictionary containing all the recovery parameters :param str remote_command: ssh command for remote connection :param tuple required_xlog_files: list of required WAL segments """ if remote_command: status_dir = recovery_info['tempdir'] else: status_dir = os.path.join(recovery_info['wal_dest'], 'archive_status') mkpath(status_dir) for wal_info in required_xlog_files: with open(os.path.join(status_dir, "%s.done" % wal_info.name), 'a') as f: f.write('') if remote_command: try: recovery_info['rsync']('%s/' % status_dir, ':%s' % os.path.join( recovery_info['wal_dest'], 'archive_status')) except CommandFailedException as e: output.error("unable to populate archive_status " "directory: %s", e) output.close_and_exit() def _generate_recovery_conf(self, recovery_info, backup_info, dest, immediate, exclusive, remote_command, target_name, target_time, target_tli, target_xid): """ Generate a recovery.conf file for PITR containing all the required configurations :param dict recovery_info: Dictionary containing all the recovery parameters :param barman.infofile.BackupInfo backup_info: representation of a backup :param str dest: destination directory of the recovery :param str|None immediate: end recovery as soon as consistency is reached :param boolean exclusive: exclusive backup or concurrent :param str remote_command: ssh command for remote connection :param str target_name: recovery target name for PITR :param str target_time: recovery target time for PITR :param str target_tli: recovery target timeline for PITR :param str target_xid: recovery target transaction id for PITR """ if remote_command: recovery = open(os.path.join(recovery_info['tempdir'], 'recovery.conf'), 'w') else: recovery = open(os.path.join(dest, 'recovery.conf'), 'w') # If GET_WAL has been set, use the get-wal command to retrieve the # required wal files. Otherwise use the unix command "cp" to copy # them from the barman_xlog directory if recovery_info['get_wal']: # We need to create the right restore command. # If we are doing a remote recovery, # the barman-cli package is REQUIRED on the server that is hosting # the PostgreSQL server. # We use the machine FQDN and the barman_user # setting to call the barman-wal-restore correctly. # If local recovery, we use barman directly, assuming # the postgres process will be executed with the barman user. # It MUST to be reviewed by the user in any case. if remote_command: fqdn = socket.getfqdn() print("# The 'barman-wal-restore' command " "is provided in the 'barman-cli' package", file=recovery) print("restore_command = 'barman-wal-restore -U %s " "%s %s %%f %%p'" % (self.config.config.user, fqdn, self.config.name), file=recovery) else: print("# The 'barman get-wal' command " "must run as '%s' user" % self.config.config.user, file=recovery) print("restore_command = 'sudo -u %s " "barman get-wal %s %%f > %%p'" % ( self.config.config.user, self.config.name), file=recovery) recovery_info['results']['get_wal'] = True else: print("restore_command = 'cp barman_xlog/%f %p'", file=recovery) if backup_info.version >= 80400 and \ not recovery_info['get_wal']: print("recovery_end_command = 'rm -fr barman_xlog'", file=recovery) # Writes recovery target if target_time: print("recovery_target_time = '%s'" % target_time, file=recovery) if target_xid: print("recovery_target_xid = '%s'" % target_xid, file=recovery) if target_name: print("recovery_target_name = '%s'" % target_name, file=recovery) # TODO: log a warning if PostgreSQL < 9.4 and --immediate if (backup_info.version >= 90400 and immediate): print("recovery_target = 'immediate'", file=recovery) # Manage what happens after recovery target is reached if (target_xid or target_time) and exclusive: print("recovery_target_inclusive = '%s'" % ( not exclusive), file=recovery) if target_tli: print("recovery_target_timeline = %s" % target_tli, file=recovery) recovery.close() if remote_command: plain_rsync = RsyncPgData( path=self.server.path, ssh=remote_command, bwlimit=self.config.bandwidth_limit, network_compression=self.config.network_compression) try: plain_rsync.from_file_list(['recovery.conf'], recovery_info['tempdir'], ':%s' % dest) except CommandFailedException as e: output.error('remote copy of recovery.conf failed: %s', e) output.close_and_exit() def _map_temporary_config_files(self, recovery_info, backup_info, remote_command): """ Map configuration files, by filling the 'temporary_configuration_files' array, depending on remote or local recovery. This array will be used by the subsequent methods of the class. :param dict recovery_info: Dictionary containing all the recovery parameters :param barman.infofile.BackupInfo backup_info: a backup representation :param str remote_command: ssh command for remote recovery """ # Cycle over postgres configuration files which my be missing. # If a file is missing, we will be unable to restore it and # we will warn the user. # This can happen if we are using pg_basebackup and # a configuration file is located outside the data dir. # This is not an error condition, so we check also for # `pg_ident.conf` which is an optional file. for conf_file in (recovery_info['configuration_files'] + ['pg_hba.conf', 'pg_ident.conf']): source_path = os.path.join( backup_info.get_data_directory(), conf_file) if not os.path.exists(source_path): recovery_info['results']['missing_files'].append(conf_file) # Remove the file from the list of configuration files if conf_file in recovery_info['configuration_files']: recovery_info['configuration_files'].remove(conf_file) for conf_file in recovery_info['configuration_files']: if remote_command: # If the recovery is remote, copy the postgresql.conf # file in a temp dir # Otherwise we can modify the postgresql.conf file # in the destination directory. conf_file_path = os.path.join( recovery_info['tempdir'], conf_file) shutil.copy2( os.path.join(backup_info.get_data_directory(), conf_file), conf_file_path) else: # Otherwise use the local destination path. conf_file_path = os.path.join( recovery_info['destination_path'], conf_file) recovery_info['temporary_configuration_files'].append( conf_file_path) def _analyse_temporary_config_files(self, recovery_info): """ Analyse temporary configuration files and identify dangerous options Mark all the dangerous options for the user to review. This procedure also changes harmful options such as 'archive_command'. :param dict recovery_info: dictionary holding all recovery parameters """ results = recovery_info['results'] # Check for dangerous options inside every config file for conf_file in recovery_info['temporary_configuration_files']: # Identify and comment out dangerous options, replacing them with # the appropriate values results['changes'] += self._pg_config_mangle( conf_file, self.MANGLE_OPTIONS, "%s.origin" % conf_file) # Identify dangerous options and warn users about their presence results['warnings'] += self._pg_config_detect_possible_issues( conf_file) def _copy_temporary_config_files(self, dest, remote_command, recovery_info): """ Copy modified configuration files using rsync in case of remote recovery :param str dest: destination directory of the recovery :param str remote_command: ssh command for remote connection :param dict recovery_info: Dictionary containing all the recovery parameters """ if remote_command: # If this is a remote recovery, rsync the modified files from the # temporary local directory to the remote destination directory. file_list = [] for conf_file in recovery_info['configuration_files']: file_list.append('%s' % conf_file) file_list.append('%s.origin' % conf_file) try: recovery_info['rsync'].from_file_list(file_list, recovery_info['tempdir'], ':%s' % dest) except CommandFailedException as e: output.error('remote copy of configuration files failed: %s', e) output.close_and_exit() def _teardown(self, recovery_info): """ Cleanup operations for a recovery :param dict recovery_info: dictionary holding the basic values for a recovery """ # Remove the temporary directory (created in the setup method) shutil.rmtree(recovery_info['tempdir']) def _pg_config_mangle(self, filename, settings, backup_filename=None): """ This method modifies the given PostgreSQL configuration file, commenting out the given settings, and adding the ones generated by Barman. If backup_filename is passed, keep a backup copy. :param filename: the PostgreSQL configuration file :param settings: dictionary of settings to be mangled :param backup_filename: config file backup copy. Default is None. """ # Read the full content of the file in memory with open(filename) as f: content = f.readlines() # Rename the original file to backup_filename or to a temporary name # if backup_filename is missing. We need to keep it to preserve # permissions. if backup_filename: orig_filename = backup_filename else: orig_filename = "%s.config_mangle.old" % filename shutil.move(filename, orig_filename) # Write the mangled content mangled = [] with open(filename, 'w') as f: for l_number, line in enumerate(content): rm = PG_CONF_SETTING_RE.match(line) if rm: key = rm.group(1) if key in settings: f.write("#BARMAN# %s" % line) # TODO is it useful to handle none values? changes = "%s = %s\n" % (key, settings[key]) f.write(changes) mangled.append( Assertion._make([ os.path.basename(f.name), l_number, key, settings[key]])) continue f.write(line) # Restore original permissions shutil.copymode(orig_filename, filename) # If a backup copy of the file is not requested, # unlink the orig file if not backup_filename: os.unlink(orig_filename) return mangled def _pg_config_detect_possible_issues(self, filename): """ This method looks for any possible issue with PostgreSQL location options such as data_directory, config_file, etc. It returns a dictionary with the dangerous options that have been found. :param filename: the Postgres configuration file """ clashes = [] with open(filename) as f: content = f.readlines() # Read line by line and identify dangerous options for l_number, line in enumerate(content): rm = PG_CONF_SETTING_RE.match(line) if rm: key = rm.group(1) if key in self.DANGEROUS_OPTIONS: clashes.append( Assertion._make([ os.path.basename(f.name), l_number, key, rm.group(2)])) return clashes barman-2.3/barman/remote_status.py0000644000076500000240000000436013134472625020075 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ Remote Status module A Remote Status class implements a standard interface for retrieving and caching the results of a remote component (such as Postgres server, WAL archiver, etc.). It follows the Mixin pattern. """ from abc import ABCMeta, abstractmethod from barman.utils import with_metaclass class RemoteStatusMixin(with_metaclass(ABCMeta, object)): """ Abstract base class that implements remote status capabilities following the Mixin pattern. """ def __init__(self, *args, **kwargs): """ Base constructor (Mixin pattern) """ self._remote_status = None super(RemoteStatusMixin, self).__init__(*args, **kwargs) @abstractmethod def fetch_remote_status(self): """ Retrieve status information from the remote component The implementation of this method must not raise any exception in case of errors, but should set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ def get_remote_status(self): """ Get the status of the remote component This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ if self._remote_status is None: self._remote_status = self.fetch_remote_status() return self._remote_status def reset_remote_status(self): """ Reset the cached result """ self._remote_status = None barman-2.3/barman/retention_policies.py0000644000076500000240000003425513134472625021103 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module defines backup retention policies. A backup retention policy in Barman is a user-defined policy for determining how long backups and archived logs (WAL segments) need to be retained for media recovery. You can define a retention policy in terms of backup redundancy or a recovery window. Barman retains the periodical backups required to satisfy the current retention policy, and any archived WAL files required for complete recovery of those backups. """ import logging import re from abc import ABCMeta, abstractmethod from datetime import datetime, timedelta from dateutil import tz from barman.infofile import BackupInfo from barman.utils import with_metaclass _logger = logging.getLogger(__name__) class RetentionPolicy(with_metaclass(ABCMeta, object)): """Abstract base class for retention policies""" def __init__(self, mode, unit, value, context, server): """Constructor of the retention policy base class""" self.mode = mode self.unit = unit self.value = int(value) self.context = context self.server = server self._first_backup = None self._first_wal = None def report(self, source=None, context=None): """Report obsolete/valid objects according to the retention policy""" if context is None: context = self.context # Overrides the list of available backups if source is None: source = self.server.get_available_backups( BackupInfo.STATUS_NOT_EMPTY) if context == 'BASE': return self._backup_report(source) elif context == 'WAL': return self._wal_report() else: raise ValueError('Invalid context %s', context) def backup_status(self, backup_id): """Report the status of a backup according to the retention policy""" source = self.server.get_available_backups(BackupInfo.STATUS_NOT_EMPTY) if self.context == 'BASE': return self._backup_report(source)[backup_id] else: return BackupInfo.NONE def first_backup(self): """Returns the first valid backup according to retention policies""" if not self._first_backup: self.report(context='BASE') return self._first_backup def first_wal(self): """Returns the first valid WAL according to retention policies""" if not self._first_wal: self.report(context='WAL') return self._first_wal @abstractmethod def __str__(self): """String representation""" pass @abstractmethod def debug(self): """Debug information""" pass @abstractmethod def _backup_report(self, source): """Report obsolete/valid backups according to the retention policy""" pass @abstractmethod def _wal_report(self): """Report obsolete/valid WALs according to the retention policy""" pass @classmethod def create(cls, server, option, value): """ If given option and value from the configuration file match, creates the retention policy object for the given server """ # using @abstractclassmethod from python3 would be better here raise NotImplementedError( 'The class %s must override the create() class method', cls.__name__) def to_json(self): """ Output representation of the obj for JSON serialization """ return "%s %s %s" % (self.mode, self.value, self.unit) class RedundancyRetentionPolicy(RetentionPolicy): """ Retention policy based on redundancy, the setting that determines many periodical backups to keep. A redundancy-based retention policy is contrasted with retention policy that uses a recovery window. """ _re = re.compile(r'^\s*redundancy\s+(\d+)\s*$', re.IGNORECASE) def __init__(self, context, value, server): super(RedundancyRetentionPolicy, self ).__init__('redundancy', 'b', value, 'BASE', server) assert (value >= 0) def __str__(self): return "REDUNDANCY %s" % self.value def debug(self): return "Redundancy: %s (%s)" % (self.value, self.context) def _backup_report(self, source): """Report obsolete/valid backups according to the retention policy""" report = dict() backups = source # Normalise the redundancy value (according to minimum redundancy) redundancy = self.value if redundancy < self.server.config.minimum_redundancy: _logger.warning( "Retention policy redundancy (%s) is lower than " "the required minimum redundancy (%s). Enforce %s.", redundancy, self.server.config.minimum_redundancy, self.server.config.minimum_redundancy) redundancy = self.server.config.minimum_redundancy # Map the latest 'redundancy' DONE backups as VALID # The remaining DONE backups are classified as OBSOLETE # Non DONE backups are classified as NONE # NOTE: reverse key orders (simulate reverse chronology) i = 0 for bid in sorted(backups.keys(), reverse=True): if backups[bid].status == BackupInfo.DONE: if i < redundancy: report[bid] = BackupInfo.VALID self._first_backup = bid else: report[bid] = BackupInfo.OBSOLETE i = i + 1 else: report[bid] = BackupInfo.NONE return report def _wal_report(self): """Report obsolete/valid WALs according to the retention policy""" pass @classmethod def create(cls, server, context, optval): # Detect Redundancy retention type mtch = cls._re.match(optval) if not mtch: return None value = int(mtch.groups()[0]) return cls(context, value, server) class RecoveryWindowRetentionPolicy(RetentionPolicy): """ Retention policy based on recovery window. The DBA specifies a period of time and Barman ensures retention of backups and archived WAL files required for point-in-time recovery to any time during the recovery window. The interval always ends with the current time and extends back in time for the number of days specified by the user. For example, if the retention policy is set for a recovery window of seven days, and the current time is 9:30 AM on Friday, Barman retains the backups required to allow point-in-time recovery back to 9:30 AM on the previous Friday. """ _re = re.compile( r""" ^\s* recovery\s+window\s+of\s+ # recovery window of (\d+)\s+(day|month|week)s? # N (day|month|week) with optional 's' \s*$ """, re.IGNORECASE | re.VERBOSE) _kw = {'d': 'DAYS', 'm': 'MONTHS', 'w': 'WEEKS'} def __init__(self, context, value, unit, server): super(RecoveryWindowRetentionPolicy, self ).__init__('window', unit, value, context, server) assert (value >= 0) assert (unit == 'd' or unit == 'm' or unit == 'w') assert (context == 'WAL' or context == 'BASE') # Calculates the time delta if unit == 'd': self.timedelta = timedelta(days=self.value) elif unit == 'w': self.timedelta = timedelta(weeks=self.value) elif unit == 'm': self.timedelta = timedelta(days=(31 * self.value)) def __str__(self): return "RECOVERY WINDOW OF %s %s" % (self.value, self._kw[self.unit]) def debug(self): return "Recovery Window: %s %s: %s (%s)" % ( self.value, self.unit, self.context, self._point_of_recoverability()) def _point_of_recoverability(self): """ Based on the current time and the window, calculate the point of recoverability, which will be then used to define the first backup or the first WAL """ return datetime.now(tz.tzlocal()) - self.timedelta def _backup_report(self, source): """Report obsolete/valid backups according to the retention policy""" report = dict() backups = source # Map as VALID all DONE backups having end time lower than # the point of recoverability. The older ones # are classified as OBSOLETE. # Non DONE backups are classified as NONE found = False valid = 0 # NOTE: reverse key orders (simulate reverse chronology) for bid in sorted(backups.keys(), reverse=True): # We are interested in DONE backups only if backups[bid].status == BackupInfo.DONE: if found: # Check minimum redundancy requirements if valid < self.server.config.minimum_redundancy: _logger.warning( "Keeping obsolete backup %s for server %s " "(older than %s) " "due to minimum redundancy requirements (%s)", bid, self.server.config.name, self._point_of_recoverability(), self.server.config.minimum_redundancy) # We mark the backup as potentially obsolete # as we must respect minimum redundancy requirements report[bid] = BackupInfo.POTENTIALLY_OBSOLETE self._first_backup = bid valid = valid + 1 else: # We mark this backup as obsolete # (older than the first valid one) _logger.info( "Reporting backup %s for server %s as OBSOLETE " "(older than %s)", bid, self.server.config.name, self._point_of_recoverability()) report[bid] = BackupInfo.OBSOLETE else: _logger.debug( "Reporting backup %s for server %s as VALID " "(newer than %s)", bid, self.server.config.name, self._point_of_recoverability()) # Backup within the recovery window report[bid] = BackupInfo.VALID self._first_backup = bid valid = valid + 1 # TODO: Currently we use the backup local end time # We need to make this more accurate if backups[bid].end_time < self._point_of_recoverability(): found = True else: report[bid] = BackupInfo.NONE return report def _wal_report(self): """Report obsolete/valid WALs according to the retention policy""" pass @classmethod def create(cls, server, context, optval): # Detect Recovery Window retention type match = cls._re.match(optval) if not match: return None value = int(match.groups()[0]) unit = match.groups()[1][0].lower() return cls(context, value, unit, server) class SimpleWALRetentionPolicy(RetentionPolicy): """Simple retention policy for WAL files (identical to the main one)""" _re = re.compile(r'^\s*main\s*$', re.IGNORECASE) def __init__(self, context, policy, server): super(SimpleWALRetentionPolicy, self ).__init__('simple-wal', policy.unit, policy.value, context, server) # The referred policy must be of type 'BASE' assert (self.context == 'WAL' and policy.context == 'BASE') self.policy = policy def __str__(self): return "MAIN" def debug(self): return "Simple WAL Retention Policy (%s)" % self.policy def _backup_report(self, source): """Report obsolete/valid backups according to the retention policy""" pass def _wal_report(self): """Report obsolete/valid backups according to the retention policy""" self.policy.report(context='WAL') def first_wal(self): """Returns the first valid WAL according to retention policies""" return self.policy.first_wal() @classmethod def create(cls, server, context, optval): # Detect Redundancy retention type match = cls._re.match(optval) if not match: return None return cls(context, server.config.retention_policy, server) class RetentionPolicyFactory(object): """Factory for retention policy objects""" # Available retention policy types policy_classes = [ RedundancyRetentionPolicy, RecoveryWindowRetentionPolicy, SimpleWALRetentionPolicy ] @classmethod def create(cls, server, option, value): """ Based on the given option and value from the configuration file, creates the appropriate retention policy object for the given server """ if option == 'wal_retention_policy': context = 'WAL' elif option == 'retention_policy': context = 'BASE' else: raise ValueError('Unknown option for retention policy: %s' % option) # Look for the matching rule for policy_class in cls.policy_classes: policy = policy_class.create(server, context, value) if policy: return policy raise ValueError('Cannot parse option %s: %s' % (option, value)) barman-2.3/barman/server.py0000644000076500000240000025050713152223257016506 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module represents a Server. Barman is able to manage multiple servers. """ import logging import os import shutil import sys import time from collections import namedtuple from contextlib import contextmanager from glob import glob from tempfile import NamedTemporaryFile import barman from barman import output, xlog from barman.backup import BackupManager from barman.command_wrappers import BarmanSubProcess from barman.compression import identify_compression from barman.exceptions import (ArchiverFailure, BadXlogSegmentName, ConninfoException, LockFileBusy, LockFilePermissionDenied, PostgresDuplicateReplicationSlot, PostgresException, PostgresInvalidReplicationSlot, PostgresIsInRecovery, PostgresReplicationSlotInUse, PostgresReplicationSlotsFull, PostgresSuperuserRequired, PostgresUnsupportedFeature, TimeoutError, UnknownBackupIdException) from barman.infofile import BackupInfo, WalFileInfo from barman.lockfile import (ServerBackupLock, ServerCronLock, ServerWalArchiveLock, ServerWalReceiveLock, ServerXLOGDBLock) from barman.postgres import PostgreSQLConnection, StreamingConnection from barman.process import ProcessManager from barman.remote_status import RemoteStatusMixin from barman.retention_policies import RetentionPolicyFactory from barman.utils import (human_readable_timedelta, is_power_of_two, pretty_size, timeout) from barman.wal_archiver import (FileWalArchiver, StreamingWalArchiver, WalArchiver) _logger = logging.getLogger(__name__) class CheckStrategy(object): """ This strategy for the 'check' collects the results of every check and does not print any message. This basic class is also responsible for immediately logging any performed check with an error in case of check failure and a debug message in case of success. """ # create a namedtuple object called CheckResult to manage check results CheckResult = namedtuple('CheckResult', 'server_name check status') # Default list used as a filter to identify non-critical checks NON_CRITICAL_CHECKS = ['minimum redundancy requirements', 'backup maximum age', 'failed backups', 'archiver errors', 'empty incoming directory', 'empty streaming directory', 'incoming WALs directory', 'streaming WALs directory', ] def __init__(self, ignore_checks=NON_CRITICAL_CHECKS): """ Silent Strategy constructor :param list ignore_checks: list of checks that can be ignored """ self.ignore_list = ignore_checks self.check_result = [] self.has_error = False self.running_check = None def init_check(self, check_name): """ Mark in the debug log when barman starts the execution of a check :param str check_name: the name of the check that is starting """ self.running_check = check_name _logger.debug("Starting check: %s" % check_name) def _check_name(self, check): if not check: check = self.running_check assert check return check def result(self, server_name, status, hint=None, check=None): """ Store the result of a check (with no output). Log any check result (error or debug level). :param str server_name: the server is being checked :param bool status: True if succeeded :param str,None hint: hint to print if not None: :param str,None check: the check name """ check = self._check_name(check) if not status: # If the name of the check is not in the filter list, # treat it as a blocking error, then notify the error # and change the status of the strategy if check not in self.ignore_list: self.has_error = True _logger.error( "Check '%s' failed for server '%s'" % (check, server_name)) else: # otherwise simply log the error (as info) _logger.info( "Ignoring failed check '%s' for server '%s'" % (check, server_name)) else: _logger.debug( "Check '%s' succeeded for server '%s'" % (check, server_name)) # Store the result and does not output anything result = self.CheckResult(server_name, check, status) self.check_result.append(result) self.running_check = None class CheckOutputStrategy(CheckStrategy): """ This strategy for the 'check' command immediately sends the result of a check to the designated output channel. This class derives from the basic CheckStrategy, reuses the same logic and adds output messages. """ def __init__(self): """ Output Strategy constructor """ super(CheckOutputStrategy, self).__init__(ignore_checks=()) def result(self, server_name, status, hint=None, check=None): """ Store the result of a check. Log any check result (error or debug level). Output the result to the user :param str server_name: the server being checked :param str check: the check name :param bool status: True if succeeded :param str,None hint: hint to print if not None: """ check = self._check_name(check) super(CheckOutputStrategy, self).result( server_name, status, hint, check) # Send result to output output.result('check', server_name, check, status, hint) class Server(RemoteStatusMixin): """ This class represents the PostgreSQL server to backup. """ XLOG_DB = "xlog.db" # the strategy for the management of the results of the various checks __default_check_strategy = CheckOutputStrategy() def __init__(self, config): """ Server constructor. :param barman.config.ServerConfig config: the server configuration """ super(Server, self).__init__() self.config = config self.path = self._build_path(self.config.path_prefix) self.process_manager = ProcessManager(self.config) self.enforce_retention_policies = False self.postgres = None self.streaming = None self.archivers = [] # Initialize the backup manager self.backup_manager = BackupManager(self) # Initialize the main PostgreSQL connection try: self.postgres = PostgreSQLConnection(config) # If the PostgreSQLConnection creation fails, disable the Server except ConninfoException as e: self.config.disabled = True self.config.msg_list.append("PostgreSQL connection: " + str(e).strip()) # ARCHIVER_OFF_BACKCOMPATIBILITY - START OF CODE # IMPORTANT: This is a back-compatibility feature that has # been added in Barman 2.0. It highlights a deprecated # behaviour, and helps users during this transition phase. # It forces 'archiver=on' when both archiver and streaming_archiver # are set to 'off' (default values) and displays a warning, # requesting users to explicitly set the value in the configuration. # When this back-compatibility feature will be removed from Barman # (in a couple of major releases), developers will need to remove # this block completely and reinstate the block of code you find # a few lines below (search for ARCHIVER_OFF_BACKCOMPATIBILITY # throughout the code). if self.config.archiver is False and \ self.config.streaming_archiver is False: output.warning("No archiver enabled for server '%s'. " "Please turn on 'archiver', 'streaming_archiver' " "or both", self.config.name) output.warning("Forcing 'archiver = on'") self.config.archiver = True # ARCHIVER_OFF_BACKCOMPATIBILITY - END OF CODE # Initialize the FileWalArchiver # WARNING: Order of items in self.archivers list is important! # The files will be archived in that order. if self.config.archiver: try: self.archivers.append(FileWalArchiver(self.backup_manager)) except AttributeError as e: _logger.debug(e) self.config.disabled = True self.config.msg_list.append('Unable to initialise the ' 'file based archiver') # Initialize the streaming PostgreSQL connection only when # backup_method is postgres or the streaming_archiver is in use if (self.config.backup_method == 'postgres' or self.config.streaming_archiver): try: self.streaming = StreamingConnection(config) # If the StreamingConnection creation fails, disable the server except ConninfoException as e: self.config.disabled = True self.config.msg_list.append("Streaming connection: " + str(e).strip()) # Initialize the StreamingWalArchiver # WARNING: Order of items in self.archivers list is important! # The files will be archived in that order. if self.config.streaming_archiver: try: self.archivers.append(StreamingWalArchiver( self.backup_manager)) # If the StreamingWalArchiver creation fails, # disable the server except AttributeError as e: _logger.debug(e) self.config.disabled = True self.config.msg_list.append('Unable to initialise the ' 'streaming archiver') # IMPORTANT: The following lines of code have been # temporarily commented in order to make the code # back-compatible after the introduction of 'archiver=off' # as default value in Barman 2.0. # When the back compatibility feature for archiver will be # removed, the following lines need to be decommented. # ARCHIVER_OFF_BACKCOMPATIBILITY - START OF CODE # # At least one of the available archive modes should be enabled # if len(self.archivers) < 1: # self.config.disabled = True # self.config.msg_list.append( # "No archiver enabled for server '%s'. " # "Please turn on 'archiver', 'streaming_archiver' or both" # % config.name) # ARCHIVER_OFF_BACKCOMPATIBILITY - END OF CODE # Sanity check: if file based archiver is disabled, and only # WAL streaming is enabled, a replication slot name must be configured. if not self.config.archiver and self.config.streaming_archiver and \ self.config.slot_name is None: self.config.disabled = True self.config.msg_list.append( "Streaming-only archiver requires 'streaming_conninfo' and " "'slot_name' options to be properly configured") # Set bandwidth_limit if self.config.bandwidth_limit: try: self.config.bandwidth_limit = int(self.config.bandwidth_limit) except ValueError: _logger.warning('Invalid bandwidth_limit "%s" for server "%s" ' '(fallback to "0")' % ( self.config.bandwidth_limit, self.config.name)) self.config.bandwidth_limit = None # set tablespace_bandwidth_limit if self.config.tablespace_bandwidth_limit: rules = {} for rule in self.config.tablespace_bandwidth_limit.split(): try: key, value = rule.split(':', 1) value = int(value) if value != self.config.bandwidth_limit: rules[key] = value except ValueError: _logger.warning( "Invalid tablespace_bandwidth_limit rule '%s'" % rule) if len(rules) > 0: self.config.tablespace_bandwidth_limit = rules else: self.config.tablespace_bandwidth_limit = None # Set minimum redundancy (default 0) if self.config.minimum_redundancy.isdigit(): self.config.minimum_redundancy = int( self.config.minimum_redundancy) if self.config.minimum_redundancy < 0: _logger.warning('Negative value of minimum_redundancy "%s" ' 'for server "%s" (fallback to "0")' % ( self.config.minimum_redundancy, self.config.name)) self.config.minimum_redundancy = 0 else: _logger.warning('Invalid minimum_redundancy "%s" for server "%s" ' '(fallback to "0")' % ( self.config.minimum_redundancy, self.config.name)) self.config.minimum_redundancy = 0 # Initialise retention policies self._init_retention_policies() def _init_retention_policies(self): # Set retention policy mode if self.config.retention_policy_mode != 'auto': _logger.warning( 'Unsupported retention_policy_mode "%s" for server "%s" ' '(fallback to "auto")' % ( self.config.retention_policy_mode, self.config.name)) self.config.retention_policy_mode = 'auto' # If retention_policy is present, enforce them if self.config.retention_policy: # Check wal_retention_policy if self.config.wal_retention_policy != 'main': _logger.warning( 'Unsupported wal_retention_policy value "%s" ' 'for server "%s" (fallback to "main")' % ( self.config.wal_retention_policy, self.config.name)) self.config.wal_retention_policy = 'main' # Create retention policy objects try: rp = RetentionPolicyFactory.create( self, 'retention_policy', self.config.retention_policy) # Reassign the configuration value (we keep it in one place) self.config.retention_policy = rp _logger.debug('Retention policy for server %s: %s' % ( self.config.name, self.config.retention_policy)) try: rp = RetentionPolicyFactory.create( self, 'wal_retention_policy', self.config.wal_retention_policy) # Reassign the configuration value # (we keep it in one place) self.config.wal_retention_policy = rp _logger.debug( 'WAL retention policy for server %s: %s' % ( self.config.name, self.config.wal_retention_policy)) except ValueError: _logger.exception( 'Invalid wal_retention_policy setting "%s" ' 'for server "%s" (fallback to "main")' % ( self.config.wal_retention_policy, self.config.name)) rp = RetentionPolicyFactory.create( self, 'wal_retention_policy', 'main') self.config.wal_retention_policy = rp self.enforce_retention_policies = True except ValueError: _logger.exception( 'Invalid retention_policy setting "%s" for server "%s"' % ( self.config.retention_policy, self.config.name)) def close(self): """ Close all the open connections to PostgreSQL """ if self.postgres: self.postgres.close() if self.streaming: self.streaming.close() def check(self, check_strategy=__default_check_strategy): """ Implements the 'server check' command and makes sure SSH and PostgreSQL connections work properly. It checks also that backup directories exist (and if not, it creates them). The check command will time out after a time interval defined by the check_timeout configuration value (default 30 seconds) :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ try: with timeout(self.config.check_timeout): # Check WAL archive self.check_archive(check_strategy) # Check postgres configuration self.check_postgres(check_strategy) # Check barman directories from barman configuration self.check_directories(check_strategy) # Check retention policies self.check_retention_policy_settings(check_strategy) # Check for backup validity self.check_backup_validity(check_strategy) # Executes the backup manager set of checks self.backup_manager.check(check_strategy) # Check if the msg_list of the server # contains messages and output eventual failures self.check_configuration(check_strategy) # Executes check() for every archiver, passing # remote status information for efficiency for archiver in self.archivers: archiver.check(check_strategy) # Check archiver errors self.check_archiver_errors(check_strategy) except TimeoutError: # The check timed out. # Add a failed entry to the check strategy for this. _logger.debug("Check command timed out executing '%s' check" % check_strategy.running_check) check_strategy.result(self.config.name, False, hint='barman check command timed out', check='check timeout') def check_archive(self, check_strategy): """ Checks WAL archive :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check("WAL archive") # Make sure that WAL archiving has been setup # XLOG_DB needs to exist and its size must be > 0 # NOTE: we do not need to acquire a lock in this phase xlogdb_empty = True if os.path.exists(self.xlogdb_file_name): with open(self.xlogdb_file_name, "rb") as fxlogdb: if os.fstat(fxlogdb.fileno()).st_size > 0: xlogdb_empty = False # NOTE: This check needs to be only visible if it fails if xlogdb_empty: check_strategy.result( self.config.name, False, hint='please make sure WAL shipping is setup') # Check the number of wals in the incoming directory self._check_wal_queue(check_strategy, 'incoming', 'archiver') # Check the number of wals in the streaming directory self._check_wal_queue(check_strategy, 'streaming', 'streaming_archiver') def _check_wal_queue(self, check_strategy, dir_name, archiver_name): """ Check if one of the wal queue directories beyond the max file threshold """ # Read the wal queue location from the configuration config_name = "%s_wals_directory" % dir_name assert hasattr(self.config, config_name) incoming_dir = getattr(self.config, config_name) # Check if the archiver is enabled assert hasattr(self.config, archiver_name) enabled = getattr(self.config, archiver_name) # Inspect the wal queue directory file_count = 0 for file_item in glob(os.path.join(incoming_dir, '*')): # Ignore temporary files if file_item.endswith('.tmp'): continue file_count += 1 max_incoming_wal = self.config.max_incoming_wals_queue # Subtract one from the count because of .partial file inside the # streaming directory if dir_name == 'streaming': file_count -= 1 # If this archiver is disabled, check the number of files in the # corresponding directory. # If the directory is NOT empty, fail the check and warn the user. # NOTE: This check is visible only when it fails check_strategy.init_check("empty %s directory" % dir_name) if not enabled: if file_count > 0: check_strategy.result( self.config.name, False, hint="'%s' must be empty when %s=off" % (incoming_dir, archiver_name)) # No more checks are required if the archiver # is not enabled return # At this point if max_wals_count is none, # means that no limit is set so we just need to return if max_incoming_wal is None: return check_strategy.init_check("%s WALs directory" % dir_name) if file_count > max_incoming_wal: msg = 'there are too many WALs in queue: ' \ '%s, max %s' % (file_count, max_incoming_wal) check_strategy.result(self.config.name, False, hint=msg) def check_postgres(self, check_strategy): """ Checks PostgreSQL connection :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('PostgreSQL') # Take the status of the remote server remote_status = self.get_remote_status() if remote_status.get('server_txt_version'): check_strategy.result(self.config.name, True) else: check_strategy.result(self.config.name, False) return # Check for superuser privileges in PostgreSQL if remote_status.get('is_superuser') is not None: check_strategy.init_check('is_superuser') if remote_status.get('is_superuser'): check_strategy.result( self.config.name, True) else: check_strategy.result( self.config.name, False, hint='superuser privileges for PostgreSQL ' 'connection required', check='not superuser' ) if 'streaming_supported' in remote_status: check_strategy.init_check("PostgreSQL streaming") hint = None # If a streaming connection is available, # add its status to the output of the check if remote_status['streaming_supported'] is None: hint = remote_status['connection_error'] elif not remote_status['streaming_supported']: hint = ('Streaming connection not supported' ' for PostgreSQL < 9.2') check_strategy.result(self.config.name, remote_status.get('streaming'), hint=hint) # Check wal_level parameter: must be different from 'minimal' # the parameter has been introduced in postgres >= 9.0 if 'wal_level' in remote_status: check_strategy.init_check("wal_level") if remote_status['wal_level'] != 'minimal': check_strategy.result( self.config.name, True) else: check_strategy.result( self.config.name, False, hint="please set it to a higher level than 'minimal'") # Check the presence and the status of the configured replication slot # This check will be skipped if `slot_name` is undefined if self.config.slot_name: check_strategy.init_check("replication slot") slot = remote_status['replication_slot'] # The streaming_archiver is enabled if self.config.streaming_archiver is True: # Error if PostgreSQL is too old if not remote_status['replication_slot_support']: check_strategy.result( self.config.name, False, hint="slot_name parameter set but PostgreSQL server " "is too old (%s < 9.4)" % remote_status['server_txt_version']) # Replication slots are supported else: # The slot is not present if slot is None: check_strategy.result( self.config.name, False, hint="replication slot '%s' doesn't exist. " "Please execute 'barman receive-wal " "--create-slot %s'" % (self.config.slot_name, self.config.name)) else: # The slot is present but not initialised if slot.restart_lsn is None: check_strategy.result( self.config.name, False, hint="slot '%s' not initialised: is " "'receive-wal' running?" % self.config.slot_name) # The slot is present but not active elif slot.active is False: check_strategy.result( self.config.name, False, hint="slot '%s' not active: is " "'receive-wal' running?" % self.config.slot_name) else: check_strategy.result(self.config.name, True) else: # If the streaming_archiver is disabled and the slot_name # option is present in the configuration, we check that # a replication slot with the specified name is NOT present # and NOT active. # NOTE: This is not a failure, just a warning. if slot is not None: if slot.restart_lsn \ is not None: slot_status = 'initialised' # Check if the slot is also active if slot.active: slot_status = 'active' # Warn the user check_strategy.result( self.config.name, True, hint="WARNING: slot '%s' is %s but not required " "by the current config" % ( self.config.slot_name, slot_status)) def _make_directories(self): """ Make backup directories in case they do not exist """ for key in self.config.KEYS: if key.endswith('_directory') and hasattr(self.config, key): val = getattr(self.config, key) if val is not None and not os.path.isdir(val): # noinspection PyTypeChecker os.makedirs(val) def check_directories(self, check_strategy): """ Checks backup directories and creates them if they do not exist :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check("directories") if not self.config.disabled: try: self._make_directories() except OSError as e: check_strategy.result(self.config.name, False, "%s: %s" % (e.filename, e.strerror)) else: check_strategy.result(self.config.name, True) def check_configuration(self, check_strategy): """ Check for error messages in the message list of the server and output eventual errors :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('configuration') if len(self.config.msg_list): check_strategy.result(self.config.name, False) for conflict_paths in self.config.msg_list: output.info("\t\t%s" % conflict_paths) def check_retention_policy_settings(self, check_strategy): """ Checks retention policy setting :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check("retention policy settings") if (self.config.retention_policy and not self.enforce_retention_policies): check_strategy.result(self.config.name, False, hint='see log') else: check_strategy.result(self.config.name, True) def check_backup_validity(self, check_strategy): """ Check if backup validity requirements are satisfied :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('backup maximum age') # first check: check backup maximum age if self.config.last_backup_maximum_age is not None: # get maximum age information backup_age = self.backup_manager.validate_last_backup_maximum_age( self.config.last_backup_maximum_age) # format the output check_strategy.result( self.config.name, backup_age[0], hint="interval provided: %s, latest backup age: %s" % ( human_readable_timedelta( self.config.last_backup_maximum_age), backup_age[1])) else: # last_backup_maximum_age provided by the user check_strategy.result( self.config.name, True, hint="no last_backup_maximum_age provided") def check_archiver_errors(self, check_strategy): """ Checks the presence of archiving errors :param CheckStrategy check_strategy: the strategy for the management of the results of the check """ check_strategy.init_check('archiver errors') if os.path.isdir(self.config.errors_directory): errors = os.listdir(self.config.errors_directory) else: errors = [] check_strategy.result( self.config.name, len(errors) == 0, hint=WalArchiver.summarise_error_files(errors) ) def status_postgres(self): """ Status of PostgreSQL server """ remote_status = self.get_remote_status() if remote_status['server_txt_version']: output.result('status', self.config.name, "pg_version", "PostgreSQL version", remote_status['server_txt_version']) else: output.result('status', self.config.name, "pg_version", "PostgreSQL version", "FAILED trying to get PostgreSQL version") return # Define the cluster state as pg_controldata do. if remote_status['is_in_recovery']: output.result('status', self.config.name, 'is_in_recovery', 'Cluster state', "in archive recovery") else: output.result('status', self.config.name, 'is_in_recovery', 'Cluster state', "in production") if remote_status['pgespresso_installed']: output.result('status', self.config.name, 'pgespresso', 'pgespresso extension', "Available") else: output.result('status', self.config.name, 'pgespresso', 'pgespresso extension', "Not available") if remote_status.get('current_size') is not None: output.result('status', self.config.name, 'current_size', 'Current data size', pretty_size(remote_status['current_size'])) if remote_status['data_directory']: output.result('status', self.config.name, "data_directory", "PostgreSQL Data directory", remote_status['data_directory']) if remote_status['current_xlog']: output.result('status', self.config.name, "current_xlog", "Current WAL segment", remote_status['current_xlog']) def status_wal_archiver(self): """ Status of WAL archiver(s) """ for archiver in self.archivers: archiver.status() def status_retention_policies(self): """ Status of retention policies enforcement """ if self.enforce_retention_policies: output.result('status', self.config.name, "retention_policies", "Retention policies", "enforced " "(mode: %s, retention: %s, WAL retention: %s)" % ( self.config.retention_policy_mode, self.config.retention_policy, self.config.wal_retention_policy)) else: output.result('status', self.config.name, "retention_policies", "Retention policies", "not enforced") def status(self): """ Implements the 'server-status' command. """ if self.config.description: output.result('status', self.config.name, "description", "Description", self.config.description) output.result('status', self.config.name, "active", "Active", self.config.active) output.result('status', self.config.name, "disabled", "Disabled", self.config.disabled) self.status_postgres() self.status_wal_archiver() self.status_retention_policies() # Executes the backup manager status info method self.backup_manager.status() def fetch_remote_status(self): """ Get the status of the remote server This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ result = {} # Merge status for a postgres connection if self.postgres: result.update(self.postgres.get_remote_status()) # Merge status for a streaming connection if self.streaming: result.update(self.streaming.get_remote_status()) # Merge status for each archiver for archiver in self.archivers: result.update(archiver.get_remote_status()) # Merge status defined by the BackupManager result.update(self.backup_manager.get_remote_status()) return result def show(self): """ Shows the server configuration """ # Populate result map with all the required keys result = self.config.to_json() remote_status = self.get_remote_status() result.update(remote_status) # Backup maximum age section if self.config.last_backup_maximum_age is not None: age = self.backup_manager.validate_last_backup_maximum_age( self.config.last_backup_maximum_age) # If latest backup is between the limits of the # last_backup_maximum_age configuration, display how old is # the latest backup. if age[0]: msg = "%s (latest backup: %s )" % \ (human_readable_timedelta( self.config.last_backup_maximum_age), age[1]) else: # If latest backup is outside the limits of the # last_backup_maximum_age configuration (or the configuration # value is none), warn the user. msg = "%s (WARNING! latest backup is %s old)" % \ (human_readable_timedelta( self.config.last_backup_maximum_age), age[1]) result['last_backup_maximum_age'] = msg else: result['last_backup_maximum_age'] = "None" output.result('show_server', self.config.name, result) def delete_backup(self, backup): """Deletes a backup :param backup: the backup to delete """ try: # Lock acquisition: if you can acquire a ServerBackupLock # it means that no backup process is running on that server, # so there is no need to check the backup status. # Simply proceed with the normal delete process. server_backup_lock = ServerBackupLock( self.config.barman_lock_directory, self.config.name) server_backup_lock.acquire(server_backup_lock.raise_if_fail, server_backup_lock.wait) server_backup_lock.release() return self.backup_manager.delete_backup(backup) except LockFileBusy: # Otherwise if the lockfile is busy, a backup process is actually # running on that server. To be sure that it's safe # to delete the backup, we must check its status and its position # in the catalogue. # If it is the first and it is STARTED or EMPTY, we are trying to # remove a running backup. This operation must be forbidden. # Otherwise, normally delete the backup. first_backup_id = self.get_first_backup_id(BackupInfo.STATUS_ALL) if backup.backup_id == first_backup_id \ and backup.status in (BackupInfo.STARTED, BackupInfo.EMPTY): output.error("Cannot delete a running backup (%s %s)" % (self.config.name, backup.backup_id)) else: return self.backup_manager.delete_backup(backup) except LockFilePermissionDenied as e: # We cannot access the lockfile. # Exit without removing the backup. output.error("Permission denied, unable to access '%s'" % e) def backup(self): """ Performs a backup for the server """ try: # Default strategy for check in backup is CheckStrategy # This strategy does not print any output - it only logs checks strategy = CheckStrategy() self.check(strategy) if strategy.has_error: output.error("Impossible to start the backup. Check the log " "for more details, or run 'barman check %s'" % self.config.name) return # check required backup directories exist self._make_directories() except OSError as e: output.error('failed to create %s directory: %s', e.filename, e.strerror) return # Make sure we are not wasting an precious streaming PostgreSQL # connection that may have been opened by the self.check() call if self.streaming: self.streaming.close() try: # lock acquisition and backup execution with ServerBackupLock(self.config.barman_lock_directory, self.config.name): self.backup_manager.backup() # Archive incoming WALs and update WAL catalogue self.archive_wal(verbose=False) except LockFileBusy: output.error("Another backup process is running") except LockFilePermissionDenied as e: output.error("Permission denied, unable to access '%s'" % e) def get_available_backups( self, status_filter=BackupManager.DEFAULT_STATUS_FILTER): """ Get a list of available backups param: status_filter: the status of backups to return, default to BackupManager.DEFAULT_STATUS_FILTER """ return self.backup_manager.get_available_backups(status_filter) def get_last_backup_id( self, status_filter=BackupManager.DEFAULT_STATUS_FILTER): """ Get the id of the latest/last backup in the catalog (if exists) :param status_filter: The status of the backup to return, default to DEFAULT_STATUS_FILTER. :return string|None: ID of the backup """ return self.backup_manager.get_last_backup_id(status_filter) def get_first_backup_id( self, status_filter=BackupManager.DEFAULT_STATUS_FILTER): """ Get the id of the oldest/first backup in the catalog (if exists) :param status_filter: The status of the backup to return, default to DEFAULT_STATUS_FILTER. :return string|None: ID of the backup """ return self.backup_manager.get_first_backup_id(status_filter) def list_backups(self): """ Lists all the available backups for the server """ retention_status = self.report_backups() backups = self.get_available_backups(BackupInfo.STATUS_ALL) for key in sorted(backups.keys(), reverse=True): backup = backups[key] backup_size = backup.size or 0 wal_size = 0 rstatus = None if backup.status == BackupInfo.DONE: try: wal_info = self.get_wal_info(backup) backup_size += wal_info['wal_size'] wal_size = wal_info['wal_until_next_size'] except BadXlogSegmentName as e: output.error( "invalid WAL segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", str(e), self.config.name) if self.enforce_retention_policies and \ retention_status[backup.backup_id] != BackupInfo.VALID: rstatus = retention_status[backup.backup_id] output.result('list_backup', backup, backup_size, wal_size, rstatus) def get_backup(self, backup_id): """ Return the backup information for the given backup id. If the backup_id is None or backup.info file doesn't exists, it returns None. :param str|None backup_id: the ID of the backup to return :rtype: BackupInfo|None """ return self.backup_manager.get_backup(backup_id) def get_previous_backup(self, backup_id): """ Get the previous backup (if any) from the catalog :param backup_id: the backup id from which return the previous """ return self.backup_manager.get_previous_backup(backup_id) def get_next_backup(self, backup_id): """ Get the next backup (if any) from the catalog :param backup_id: the backup id from which return the next """ return self.backup_manager.get_next_backup(backup_id) def get_required_xlog_files(self, backup, target_tli=None, target_time=None, target_xid=None): """ Get the xlog files required for a recovery """ begin = backup.begin_wal end = backup.end_wal # If timeline isn't specified, assume it is the same timeline # of the backup if not target_tli: target_tli, _, _ = xlog.decode_segment_name(end) with self.xlogdb() as fxlogdb: for line in fxlogdb: wal_info = WalFileInfo.from_xlogdb_line(line) # Handle .history files: add all of them to the output, # regardless of their age if xlog.is_history_file(wal_info.name): yield wal_info continue if wal_info.name < begin: continue tli, _, _ = xlog.decode_segment_name(wal_info.name) if tli > target_tli: continue yield wal_info if wal_info.name > end: end = wal_info.name if target_time and target_time < wal_info.time: break # return all the remaining history files for line in fxlogdb: wal_info = WalFileInfo.from_xlogdb_line(line) if xlog.is_history_file(wal_info.name): yield wal_info # TODO: merge with the previous def get_wal_until_next_backup(self, backup, include_history=False): """ Get the xlog files between backup and the next :param BackupInfo backup: a backup object, the starting point to retrieve WALs :param bool include_history: option for the inclusion of include_history files into the output """ begin = backup.begin_wal next_end = None if self.get_next_backup(backup.backup_id): next_end = self.get_next_backup(backup.backup_id).end_wal backup_tli, _, _ = xlog.decode_segment_name(begin) with self.xlogdb() as fxlogdb: for line in fxlogdb: wal_info = WalFileInfo.from_xlogdb_line(line) # Handle .history files: add all of them to the output, # regardless of their age, if requested (the 'include_history' # parameter is True) if xlog.is_history_file(wal_info.name): if include_history: yield wal_info continue if wal_info.name < begin: continue tli, _, _ = xlog.decode_segment_name(wal_info.name) if tli > backup_tli: continue if not xlog.is_wal_file(wal_info.name): continue if next_end and wal_info.name > next_end: break yield wal_info def get_wal_full_path(self, wal_name): """ Build the full path of a WAL for a server given the name :param wal_name: WAL file name """ # Build the path which contains the file hash_dir = os.path.join(self.config.wals_directory, xlog.hash_dir(wal_name)) # Build the WAL file full path full_path = os.path.join(hash_dir, wal_name) return full_path def get_wal_info(self, backup_info): """ Returns information about WALs for the given backup :param BackupInfo backup_info: the target backup """ begin = backup_info.begin_wal end = backup_info.end_wal # counters wal_info = dict.fromkeys( ('wal_num', 'wal_size', 'wal_until_next_num', 'wal_until_next_size', 'wal_until_next_compression_ratio', 'wal_compression_ratio'), 0) # First WAL (always equal to begin_wal) and Last WAL names and ts wal_info['wal_first'] = None wal_info['wal_first_timestamp'] = None wal_info['wal_last'] = None wal_info['wal_last_timestamp'] = None # WAL rate (default 0.0 per second) wal_info['wals_per_second'] = 0.0 for item in self.get_wal_until_next_backup(backup_info): if item.name == begin: wal_info['wal_first'] = item.name wal_info['wal_first_timestamp'] = item.time if item.name <= end: wal_info['wal_num'] += 1 wal_info['wal_size'] += item.size else: wal_info['wal_until_next_num'] += 1 wal_info['wal_until_next_size'] += item.size wal_info['wal_last'] = item.name wal_info['wal_last_timestamp'] = item.time # Calculate statistics only for complete backups # If the cron is not running for any reason, the required # WAL files could be missing if wal_info['wal_first'] and wal_info['wal_last']: # Estimate WAL ratio # Calculate the difference between the timestamps of # the first WAL (begin of backup) and the last WAL # associated to the current backup wal_info['wal_total_seconds'] = ( wal_info['wal_last_timestamp'] - wal_info['wal_first_timestamp']) if wal_info['wal_total_seconds'] > 0: wal_info['wals_per_second'] = ( float(wal_info['wal_num'] + wal_info['wal_until_next_num']) / wal_info['wal_total_seconds']) # evaluation of compression ratio for basebackup WAL files wal_info['wal_theoretical_size'] = \ wal_info['wal_num'] * float(backup_info.xlog_segment_size) try: wal_info['wal_compression_ratio'] = 1 - ( wal_info['wal_size'] / wal_info['wal_theoretical_size']) except ZeroDivisionError: wal_info['wal_compression_ratio'] = 0.0 # evaluation of compression ratio of WAL files wal_info['wal_until_next_theoretical_size'] = \ wal_info['wal_until_next_num'] * \ float(backup_info.xlog_segment_size) try: wal_info['wal_until_next_compression_ratio'] = 1 - ( wal_info['wal_until_next_size'] / wal_info['wal_until_next_theoretical_size']) except ZeroDivisionError: wal_info['wal_until_next_compression_ratio'] = 0.0 return wal_info def recover(self, backup_info, dest, tablespaces=None, target_tli=None, target_time=None, target_xid=None, target_name=None, target_immediate=False, exclusive=False, remote_command=None): """ Performs a recovery of a backup :param barman.infofile.BackupInfo backup_info: the backup to recover :param str dest: the destination directory :param dict[str,str]|None tablespaces: a tablespace name -> location map (for relocation) :param str|None target_tli: the target timeline :param str|None target_time: the target time :param str|None target_xid: the target xid :param str|None target_name: the target name created previously with pg_create_restore_point() function call :param bool|None target_immediate: end recovery as soon as consistency is reached :param bool exclusive: whether the recovery is exclusive or not :param str|None remote_command: default None. The remote command to recover the base backup, in case of remote backup. """ return self.backup_manager.recover( backup_info, dest, tablespaces, target_tli, target_time, target_xid, target_name, target_immediate, exclusive, remote_command) def get_wal(self, wal_name, compression=None, output_directory=None, peek=None): """ Retrieve a WAL file from the archive :param str wal_name: id of the WAL file to find into the WAL archive :param str|None compression: compression format for the output :param str|None output_directory: directory where to deposit the WAL file :param int|None peek: if defined list the next N WAL file """ # If used through SSH identify the client to add it to logs source_suffix = '' ssh_connection = os.environ.get('SSH_CONNECTION') if ssh_connection: # The client IP is the first value contained in `SSH_CONNECTION` # which contains four space-separated values: client IP address, # client port number, server IP address, and server port number. source_suffix = ' (SSH host: %s)' % (ssh_connection.split()[0],) # Sanity check if not xlog.is_any_xlog_file(wal_name): output.error("'%s' is not a valid wal file name%s", wal_name, source_suffix) return # If peek is requested we only output a list of files if peek: # Get the next ``peek`` files following the provided ``wal_name``. # If ``wal_name`` is not a simple wal file, # we cannot guess the names of the following WAL files. # So ``wal_name`` is the only possible result, if exists. if xlog.is_wal_file(wal_name): # We can't know what was the segment size of PostgreSQL WAL # files at backup time. Because of this, we generate all # the possible names for a WAL segment, and then we check # if the requested one is included. wal_peek_list = xlog.generate_segment_names(wal_name) else: wal_peek_list = iter([wal_name]) # Output the content of wal_peek_list until we have displayed # enough files or find a missing file count = 0 while count < peek: try: wal_peek_name = next(wal_peek_list) except StopIteration: # No more item in wal_peek_list break wal_peek_file = self.get_wal_full_path(wal_peek_name) # If the next WAL file is found, output the name # and continue to the next one if os.path.exists(wal_peek_file): count += 1 output.info(wal_peek_name, log=False) continue # If ``wal_peek_file`` doesn't exist, check if we need to # look in the following segment tli, log, seg = xlog.decode_segment_name(wal_peek_name) # If `seg` is not a power of two, it is not possible that we # are at the end of a WAL group, so we are done if not is_power_of_two(seg): break # This is a possible WAL group boundary, let's try the # following group seg = 0 log += 1 # Install a new generator from the start of the next segment. # If the file doesn't exists we will terminate because # zero is not a power of two wal_peek_name = xlog.encode_segment_name(tli, log, seg) wal_peek_list = xlog.generate_segment_names(wal_peek_name) # Do not output anything else return # Get the WAL file full path wal_file = self.get_wal_full_path(wal_name) # Check for file existence if not os.path.exists(wal_file): output.error("WAL file '%s' not found in server '%s'%s", wal_name, self.config.name, source_suffix) return # If an output directory was provided write the file inside it # otherwise we use standard output if output_directory is not None: destination_path = os.path.join(output_directory, wal_name) try: destination = open(destination_path, 'w') output.info( "Writing WAL '%s' for server '%s' into '%s' file%s", wal_name, self.config.name, destination_path, source_suffix) except IOError as e: output.error("Unable to open '%s' file%s: %s", destination_path, source_suffix, e) return else: destination = sys.stdout _logger.info( "Writing WAL '%s' for server '%s' to standard output%s", wal_name, self.config.name, source_suffix) # Get a decompressor for the file (None if not compressed) wal_compressor = self.backup_manager.compression_manager \ .get_compressor(compression=identify_compression(wal_file)) # Get a compressor for the output (None if not compressed) # Here we need to handle explicitly the None value because we don't # want it ot fallback to the configured compression if compression is not None: out_compressor = self.backup_manager.compression_manager \ .get_compressor(compression=compression) else: out_compressor = None # Initially our source is the stored WAL file and we do not have # any temporary file source_file = wal_file uncompressed_file = None compressed_file = None # If the required compression is different from the source we # decompress/compress it into the required format (getattr is # used here to gracefully handle None objects) if getattr(wal_compressor, 'compression', None) != \ getattr(out_compressor, 'compression', None): # If source is compressed, decompress it into a temporary file if wal_compressor is not None: uncompressed_file = NamedTemporaryFile( dir=self.config.wals_directory, prefix='.%s.' % wal_name, suffix='.uncompressed') # decompress wal file wal_compressor.decompress(source_file, uncompressed_file.name) source_file = uncompressed_file.name # If output compression is required compress the source # into a temporary file if out_compressor is not None: compressed_file = NamedTemporaryFile( dir=self.config.wals_directory, prefix='.%s.' % wal_name, suffix='.compressed') out_compressor.compress(source_file, compressed_file.name) source_file = compressed_file.name # Copy the prepared source file to destination with open(source_file) as input_file: shutil.copyfileobj(input_file, destination) # Remove temp files if uncompressed_file is not None: uncompressed_file.close() if compressed_file is not None: compressed_file.close() def cron(self, wals=True, retention_policies=True): """ Maintenance operations :param bool wals: WAL archive maintenance :param bool retention_policies: retention policy maintenance """ try: # Actually this is the highest level of locking in the cron, # this stops the execution of multiple cron on the same server with ServerCronLock(self.config.barman_lock_directory, self.config.name): # WAL management and maintenance if wals: # Execute the archive-wal sub-process self.cron_archive_wal() if self.config.streaming_archiver: # Spawn the receive-wal sub-process self.cron_receive_wal() else: # Terminate the receive-wal sub-process if present self.kill('receive-wal', fail_if_not_present=False) # Retention policies execution if retention_policies: self.backup_manager.cron_retention_policy() except LockFileBusy: output.info( "Another cron process is already running on server %s. " "Skipping to the next server" % self.config.name) except LockFilePermissionDenied as e: output.error("Permission denied, unable to access '%s'" % e) except (OSError, IOError) as e: output.error("%s", e) def cron_archive_wal(self): """ Method that handles the start of an 'archive-wal' sub-process. This method must be run protected by ServerCronLock """ try: # Try to acquire ServerWalArchiveLock, if the lock is available, # no other 'archive-wal' processes are running on this server. # # There is a very little race condition window here because # even if we are protected by ServerCronLock, the user could run # another 'archive-wal' command manually. However, it would result # in one of the two commands failing on lock acquisition, # with no other consequence. with ServerWalArchiveLock( self.config.barman_lock_directory, self.config.name): # Output and release the lock immediately output.info("Starting WAL archiving for server %s", self.config.name, log=False) # Init a Barman sub-process object archive_process = BarmanSubProcess( subcommand='archive-wal', config=barman.__config__.config_file, args=[self.config.name]) # Launch the sub-process archive_process.execute() except LockFileBusy: # Another archive process is running for the server, # warn the user and skip to the next sever. output.info( "Another archive-wal process is already running " "on server %s. Skipping to the next server" % self.config.name) def cron_receive_wal(self): """ Method that handles the start of a 'receive-wal' sub process This method must be run protected by ServerCronLock """ try: # Try to acquire ServerWalReceiveLock, if the lock is available, # no other 'receive-wal' processes are running on this server. # # There is a very little race condition window here because # even if we are protected by ServerCronLock, the user could run # another 'receive-wal' command manually. However, it would result # in one of the two commands failing on lock acquisition, # with no other consequence. with ServerWalReceiveLock( self.config.barman_lock_directory, self.config.name): # Output and release the lock immediately output.info("Starting streaming archiver " "for server %s", self.config.name, log=False) # Start a new receive-wal process receive_process = BarmanSubProcess( subcommand='receive-wal', config=barman.__config__.config_file, args=[self.config.name]) # Launch the sub-process receive_process.execute() except LockFileBusy: # Another receive-wal process is running for the server # exit without message _logger.debug("Another STREAMING ARCHIVER process is running for " "server %s" % self.config.name) def archive_wal(self, verbose=True): """ Perform the WAL archiving operations. Usually run as subprocess of the barman cron command, but can be executed manually using the barman archive-wal command :param bool verbose: if false outputs something only if there is at least one file """ output.debug("Starting archive-wal for server %s", self.config.name) try: # Take care of the archive lock. # Only one archive job per server is admitted with ServerWalArchiveLock(self.config.barman_lock_directory, self.config.name): self.backup_manager.archive_wal(verbose) except LockFileBusy: # If another process is running for this server, # warn the user and skip to the next server output.info("Another archive-wal process is already running " "on server %s. Skipping to the next server" % self.config.name) def create_physical_repslot(self): """ Create a physical replication slot using the streaming connection """ if not self.streaming: output.error("Unable to create a physical replication slot: " "streaming connection not configured") return # Replication slots are not supported by PostgreSQL < 9.4 try: if self.streaming.server_version < 90400: output.error("Unable to create a physical replication slot: " "not supported by '%s' " "(9.4 or higher is required)" % self.streaming.server_major_version) return except PostgresException as exc: msg = "Cannot connect to server '%s'" % self.config.name output.error(msg, log=False) _logger.error("%s: %s", msg, str(exc).strip()) return if not self.config.slot_name: output.error("Unable to create a physical replication slot: " "slot_name configuration option required") return output.info( "Creating physical replication slot '%s' on server '%s'", self.config.slot_name, self.config.name) try: self.streaming.create_physical_repslot(self.config.slot_name) output.info("Replication slot '%s' created", self.config.slot_name) except PostgresDuplicateReplicationSlot: output.error("Replication slot '%s' already exists", self.config.slot_name) except PostgresReplicationSlotsFull: output.error("All replication slots for server '%s' are in use\n" "Free one or increase the max_replication_slots " "value on your PostgreSQL server.", self.config.name) except PostgresException as exc: output.error( "Cannot create replication slot '%s' on server '%s': %s", self.config.slot_name, self.config.name, str(exc).strip()) def drop_repslot(self): """ Drop a replication slot using the streaming connection """ if not self.streaming: output.error("Unable to drop a physical replication slot: " "streaming connection not configured") return # Replication slots are not supported by PostgreSQL < 9.4 try: if self.streaming.server_version < 90400: output.error("Unable to drop a physical replication slot: " "not supported by '%s' (9.4 or higher is " "required)" % self.streaming.server_major_version) return except PostgresException as exc: msg = "Cannot connect to server '%s'" % self.config.name output.error(msg, log=False) _logger.error("%s: %s", msg, str(exc).strip()) return if not self.config.slot_name: output.error("Unable to drop a physical replication slot: " "slot_name configuration option required") return output.info( "Dropping physical replication slot '%s' on server '%s'", self.config.slot_name, self.config.name) try: self.streaming.drop_repslot(self.config.slot_name) output.info("Replication slot '%s' dropped", self.config.slot_name) except PostgresInvalidReplicationSlot: output.error("Replication slot '%s' does not exist", self.config.slot_name) except PostgresReplicationSlotInUse as exc: output.error( "Cannot drop replication slot '%s' on server '%s' " "because it is in use.", self.config.slot_name, self.config.name) except PostgresException as exc: output.error( "Cannot drop replication slot '%s' on server '%s': %s", self.config.slot_name, self.config.name, str(exc).strip()) def receive_wal(self, reset=False): """ Enable the reception of WAL files using streaming protocol. Usually started by barman cron command. Executing this manually, the barman process will not terminate but will continuously receive WAL files from the PostgreSQL server. :param reset: When set, resets the status of receive-wal """ # Execute the receive-wal command only if streaming_archiver # is enabled if not self.config.streaming_archiver: output.error("Unable to start receive-wal process: " "streaming_archiver option set to 'off' in " "barman configuration file") return output.info("Starting receive-wal for server %s", self.config.name) try: # Take care of the receive-wal lock. # Only one receiving process per server is permitted with ServerWalReceiveLock(self.config.barman_lock_directory, self.config.name): try: # Only the StreamingWalArchiver implementation # does something. # WARNING: This codes assumes that there is only one # StreamingWalArchiver in the archivers list. for archiver in self.archivers: archiver.receive_wal(reset) except ArchiverFailure as e: output.error(e) except LockFileBusy: # If another process is running for this server, if reset: output.info("Unable to reset the status of receive-wal " "for server %s. Process is still running" % self.config.name) else: output.info("Another receive-wal process is already running " "for server %s." % self.config.name) @property def xlogdb_file_name(self): """ The name of the file containing the XLOG_DB :return str: the name of the file that contains the XLOG_DB """ return os.path.join(self.config.wals_directory, self.XLOG_DB) @contextmanager def xlogdb(self, mode='r'): """ Context manager to access the xlogdb file. This method uses locking to make sure only one process is accessing the database at a time. The database file will be created if it not exists. Usage example: with server.xlogdb('w') as file: file.write(new_line) :param str mode: open the file with the required mode (default read-only) """ if not os.path.exists(self.config.wals_directory): os.makedirs(self.config.wals_directory) xlogdb = self.xlogdb_file_name with ServerXLOGDBLock(self.config.barman_lock_directory, self.config.name): # If the file doesn't exist and it is required to read it, # we open it in a+ mode, to be sure it will be created if not os.path.exists(xlogdb) and mode.startswith('r'): if '+' not in mode: mode = "a%s+" % mode[1:] else: mode = "a%s" % mode[1:] with open(xlogdb, mode) as f: # execute the block nested in the with statement try: yield f finally: # we are exiting the context # if file is writable (mode contains w, a or +) # make sure the data is written to disk # http://docs.python.org/2/library/os.html#os.fsync if any((c in 'wa+') for c in f.mode): f.flush() os.fsync(f.fileno()) def report_backups(self): if not self.enforce_retention_policies: return dict() else: return self.config.retention_policy.report() def rebuild_xlogdb(self): """ Rebuild the whole xlog database guessing it from the archive content. """ return self.backup_manager.rebuild_xlogdb() def get_backup_ext_info(self, backup_info): """ Return a dictionary containing all available information about a backup The result is equivalent to the sum of information from * BackupInfo object * the Server.get_wal_info() return value * the context in the catalog (if available) * the retention policy status :param backup_info: the target backup :rtype dict: all information about a backup """ backup_ext_info = backup_info.to_dict() if backup_info.status == BackupInfo.DONE: try: previous_backup = self.backup_manager.get_previous_backup( backup_ext_info['backup_id']) next_backup = self.backup_manager.get_next_backup( backup_ext_info['backup_id']) if previous_backup: backup_ext_info[ 'previous_backup_id'] = previous_backup.backup_id else: backup_ext_info['previous_backup_id'] = None if next_backup: backup_ext_info['next_backup_id'] = next_backup.backup_id else: backup_ext_info['next_backup_id'] = None except UnknownBackupIdException: # no next_backup_id and previous_backup_id items # means "Not available" pass backup_ext_info.update(self.get_wal_info(backup_info)) if self.enforce_retention_policies: policy = self.config.retention_policy backup_ext_info['retention_policy_status'] = \ policy.backup_status(backup_info.backup_id) else: backup_ext_info['retention_policy_status'] = None # Check any child timeline exists children_timelines = self.get_children_timelines( backup_ext_info['timeline'], forked_after=backup_info.end_xlog) backup_ext_info['children_timelines'] = \ children_timelines return backup_ext_info def show_backup(self, backup_info): """ Output all available information about a backup :param backup_info: the target backup """ try: backup_ext_info = self.get_backup_ext_info(backup_info) output.result('show_backup', backup_ext_info) except BadXlogSegmentName as e: output.error( "invalid xlog segment name %r\n" "HINT: Please run \"barman rebuild-xlogdb %s\" " "to solve this issue", str(e), self.config.name) output.close_and_exit() @staticmethod def _build_path(path_prefix=None): """ If a path_prefix is provided build a string suitable to be used in PATH environment variable by joining the path_prefix with the current content of PATH environment variable. If the `path_prefix` is None returns None. :rtype: str|None """ if not path_prefix: return None sys_path = os.environ.get('PATH') return "%s%s%s" % (path_prefix, os.pathsep, sys_path) def kill(self, task, fail_if_not_present=True): """ Given the name of a barman sub-task type, attempts to stop all the processes :param string task: The task we want to stop :param bool fail_if_not_present: Display an error when the process is not present (default: True) """ process_list = self.process_manager.list(task) for process in process_list: if self.process_manager.kill(process): output.info('Stopped process %s(%s)', process.task, process.pid) return else: output.error('Cannot terminate process %s(%s)', process.task, process.pid) return if fail_if_not_present: output.error('Termination of %s failed: ' 'no such process for server %s', task, self.config.name) def switch_wal(self, force=False, archive=None, archive_timeout=None): """ Execute the switch-wal command on the target server """ try: if force: # If called with force, execute a checkpoint before the # switch_wal command _logger.info('Force a CHECKPOINT before pg_switch_wal()') self.postgres.checkpoint() # Perform the switch_wal. expect a WAL name only if the switch # has been successfully executed, False otherwise. closed_wal = self.postgres.switch_wal() if closed_wal is None: # Something went wrong during the execution of the # pg_switch_wal command output.error("Unable to perform pg_switch_wal " "for server '%s'." % self.config.name) return if closed_wal: # The switch_wal command have been executed successfully output.info( "The WAL file %s has been closed on server '%s'" % (closed_wal, self.config.name)) # If the user has asked to wait for a WAL file to be archived, # wait until a new WAL file has been found # or the timeout has expired if archive: output.info( "Waiting for the WAL file %s from server '%s' " "(max: %s seconds)", closed_wal, self.config.name, archive_timeout) # Wait for a new file until end_time end_time = time.time() + archive_timeout while time.time() < end_time: self.backup_manager.archive_wal(verbose=False) # Finish if the closed wal file is in the archive. if os.path.exists(self.get_wal_full_path(closed_wal)): break # sleep a bit before retrying time.sleep(.1) else: output.error("The WAL file %s has not been received " "in %s seconds", closed_wal, archive_timeout) else: # Is not necessary to perform a switch_wal output.info("No switch required for server '%s'" % self.config.name) except PostgresIsInRecovery: output.info("No switch performed because server '%s' " "is a standby." % self.config.name) except PostgresSuperuserRequired: # Superuser rights are required to perform the switch_wal output.error("Barman switch-wal requires superuser rights") def replication_status(self, target='all'): """ Implements the 'replication-status' command. """ if target == 'hot-standby': client_type = PostgreSQLConnection.STANDBY elif target == 'wal-streamer': client_type = PostgreSQLConnection.WALSTREAMER else: client_type = PostgreSQLConnection.ANY_STREAMING_CLIENT try: standby_info = self.postgres.get_replication_stats(client_type) if standby_info is None: output.error('Unable to connect to server %s' % self.config.name) else: output.result('replication_status', self.config.name, target, self.postgres.current_xlog_location, standby_info) except PostgresUnsupportedFeature as e: output.info(" Requires PostgreSQL %s or higher", e) except PostgresSuperuserRequired: output.info(" Requires superuser rights") def get_children_timelines(self, tli, forked_after=None): """ Get a list of the children of the passed timeline :param int tli: Id of the timeline to check :param str forked_after: XLog location after which the timeline must have been created :return List[xlog.HistoryFileData]: the list of timelines that have the timeline with id 'tli' as parent """ if forked_after: forked_after = xlog.parse_lsn(forked_after) children = [] # Search all the history files after the passed timeline children_tli = tli while True: children_tli += 1 history_path = os.path.join(self.config.wals_directory, "%08X.history" % children_tli) # If the file doesn't exists, stop searching if not os.path.exists(history_path): break # Create the WalFileInfo object using the file wal_info = WalFileInfo.from_file(history_path) # Get content of the file. We need to pass a compressor manager # here to handle an eventual compression of the history file history_info = xlog.decode_history_file( wal_info, self.backup_manager.compression_manager) # Save the history only if is reachable from this timeline. for tinfo in history_info: # The history file contains the full genealogy # but we keep only the line with `tli` timeline as parent. if tinfo.parent_tli != tli: continue # We need to return this history info only if this timeline # has been forked after the passed LSN if forked_after and tinfo.switchpoint < forked_after: continue children.append(tinfo) return children barman-2.3/barman/utils.py0000644000076500000240000002774613153276623016355 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module contains utility functions used in Barman. """ import datetime import decimal import errno import grp import json import logging import logging.handlers import os import pwd import re import signal from contextlib import contextmanager from distutils.version import Version from barman.exceptions import TimeoutError _logger = logging.getLogger(__name__) def drop_privileges(user): """ Change the system user of the current python process. It will only work if called as root or as the target user. :param string user: target user :raise KeyError: if the target user doesn't exists :raise OSError: when the user change fails """ pw = pwd.getpwnam(user) if pw.pw_uid == os.getuid(): return groups = [e.gr_gid for e in grp.getgrall() if pw.pw_name in e.gr_mem] groups.append(pw.pw_gid) os.setgroups(groups) os.setgid(pw.pw_gid) os.setuid(pw.pw_uid) os.environ['HOME'] = pw.pw_dir def mkpath(directory): """ Recursively create a target directory. If the path already exists it does nothing. :param str directory: directory to be created """ if not os.path.isdir(directory): os.makedirs(directory) def configure_logging( log_file, log_level=logging.INFO, log_format="%(asctime)s %(name)s %(levelname)s: %(message)s"): """ Configure the logging module :param str,None log_file: target file path. If None use standard error. :param int log_level: min log level to be reported in log file. Default to INFO :param str log_format: format string used for a log line. Default to "%(asctime)s %(name)s %(levelname)s: %(message)s" """ warn = None handler = logging.StreamHandler() if log_file: log_file = os.path.abspath(log_file) log_dir = os.path.dirname(log_file) try: mkpath(log_dir) handler = logging.handlers.WatchedFileHandler(log_file) except (OSError, IOError): # fallback to standard error warn = "Failed opening the requested log file. " \ "Using standard error instead." formatter = logging.Formatter(log_format) handler.setFormatter(formatter) logging.root.addHandler(handler) if warn: # this will be always displayed because the default level is WARNING _logger.warn(warn) logging.root.setLevel(log_level) def parse_log_level(log_level): """ Convert a log level to its int representation as required by logging module. :param log_level: An integer or a string :return: an integer or None if an invalid argument is provided """ try: log_level_int = int(log_level) except ValueError: log_level_int = logging.getLevelName(str(log_level).upper()) if isinstance(log_level_int, int): return log_level_int return None def pretty_size(size, unit=1024): """ This function returns a pretty representation of a size value :param int|long|float size: the number to to prettify :param int unit: 1000 or 1024 (the default) :rtype: str """ suffixes = ["B"] + [i + {1000: "B", 1024: "iB"}[unit] for i in "KMGTPEZY"] if unit == 1000: suffixes[1] = 'kB' # special case kB instead of KB # cast to float to avoid loosing decimals size = float(size) for suffix in suffixes: if abs(size) < unit or suffix == suffixes[-1]: if suffix == suffixes[0]: return "%d %s" % (size, suffix) else: return "%.1f %s" % (size, suffix) else: size /= unit def human_readable_timedelta(timedelta): """ Given a time interval, returns a human readable string :param timedelta: the timedelta to transform in a human readable form """ delta = abs(timedelta) # Calculate time units for the given interval time_map = { 'day': int(delta.days), 'hour': int(delta.seconds / 3600), 'minute': int(delta.seconds / 60) % 60, 'second': int(delta.seconds % 60), } # Build the resulting string time_list = [] # 'Day' part if time_map['day'] > 0: if time_map['day'] == 1: time_list.append('%s day' % time_map['day']) else: time_list.append('%s days' % time_map['day']) # 'Hour' part if time_map['hour'] > 0: if time_map['hour'] == 1: time_list.append('%s hour' % time_map['hour']) else: time_list.append('%s hours' % time_map['hour']) # 'Minute' part if time_map['minute'] > 0: if time_map['minute'] == 1: time_list.append('%s minute' % time_map['minute']) else: time_list.append('%s minutes' % time_map['minute']) # 'Second' part if time_map['second'] > 0: if time_map['second'] == 1: time_list.append('%s second' % time_map['second']) else: time_list.append('%s seconds' % time_map['second']) human = ', '.join(time_list) # Take care of timedelta when is shorter than a second if delta < datetime.timedelta(seconds=1): human = 'less than one second' # If timedelta is negative append 'ago' suffix if delta != timedelta: human += " ago" return human def total_seconds(timedelta): """ Compatibility method because the total_seconds method has been introduced in Python 2.7 :param timedelta: a timedelta object :rtype: float """ if hasattr(timedelta, 'total_seconds'): return timedelta.total_seconds() else: return (timedelta.microseconds + (timedelta.seconds + timedelta.days * 24 * 3600) * 10**6 ) / 10.0**6 def which(executable, path=None): """ This method is useful to find if a executable is present into the os PATH :param str executable: The name of the executable to find :param str|None path: An optional search path to override the current one. :return str|None: the path of the executable or None """ # Get the system path if needed if path is None: path = os.getenv('PATH') # If the path is None at this point we have nothing to search if path is None: return None # If executable is an absolute path, check if it exists and is executable # otherwise return failure. if os.path.isabs(executable): if os.path.exists(executable) and os.access(executable, os.X_OK): return executable else: return None # Search the requested executable in every directory present in path and # return the first occurrence that exists and is executable. for file_path in path.split(os.path.pathsep): file_path = os.path.join(file_path, executable) # If the file exists and is executable return the full path. if os.path.exists(file_path) and os.access(file_path, os.X_OK): return file_path # If no matching file is present on the system return None return None class BarmanEncoder(json.JSONEncoder): """ Custom JSON encoder used for BackupInfo encoding This encoder supports the following types: * dates and timestamps if they have a ctime() method. * objects that implement the 'to_json' method. * binary strings (python 3) """ def default(self, obj): # If the object implements to_json() method use it if hasattr(obj, 'to_json'): return obj.to_json() # Serialise date and datetime objects using ctime() method if hasattr(obj, 'ctime') and callable(obj.ctime): return obj.ctime() # Serialise timedelta objects using human_readable_timedelta() if isinstance(obj, datetime.timedelta): return human_readable_timedelta(obj) # Serialise Decimal objects using their string representation # WARNING: When deserialized they will be treat as float values # which have a lower precision if isinstance(obj, decimal.Decimal): return float(obj) # Binary strings must be decoded before using them in # an unicode string if hasattr(obj, 'decode') and callable(obj.decode): return obj.decode('utf-8', 'replace') # Manage (Loose|Strict)Version objects as strings. if isinstance(obj, Version): return str(obj) # Let the base class default method raise the TypeError return super(BarmanEncoder, self).default(obj) def fsync_dir(dir_path): """ Execute fsync on a directory ensuring it is synced to disk :param str dir_path: The directory to sync :raise OSError: If fail opening the directory """ dir_fd = os.open(dir_path, os.O_DIRECTORY) try: os.fsync(dir_fd) except OSError as e: # On some filesystem doing a fsync on a directory # raises an EINVAL error. Ignoring it is usually safe. if e.errno != errno.EINVAL: raise os.close(dir_fd) def simplify_version(version_string): """ Simplify a version number by removing the patch level :param version_string: the version number to simplify :return str: the simplified version number """ if version_string is None: return None version = version_string.split('.') # If a development/beta/rc version, split out the string part unreleased = re.search(r'[^0-9.]', version[-1]) if unreleased: last_component = version.pop() number = last_component[:unreleased.start()] string = last_component[unreleased.start():] version += [number, string] return '.'.join(version[:-1]) def with_metaclass(meta, *bases): """ Function from jinja2/_compat.py. License: BSD. Create a base class with a metaclass. :param type meta: Metaclass to add to base class """ # This requires a bit of explanation: the basic idea is to make a # dummy metaclass for one level of class instantiation that replaces # itself with the actual metaclass. class Metaclass(type): def __new__(mcs, name, this_bases, d): return meta(name, bases, d) return type.__new__(Metaclass, 'temporary_class', (), {}) @contextmanager def timeout(timeout_duration): """ ContextManager responsible for timing out the contained block of code after a defined time interval. """ # Define the handler for the alarm signal def handler(signum, frame): raise TimeoutError() # set the timeout handler previous_handler = signal.signal(signal.SIGALRM, handler) if previous_handler != signal.SIG_DFL: signal.signal(signal.SIGALRM, previous_handler) raise AssertionError("Another timeout is already defined") # set the timeout duration signal.alarm(timeout_duration) try: # Execute the contained block of code yield finally: # Reset the signal signal.alarm(0) signal.signal(signal.SIGALRM, signal.SIG_DFL) def is_power_of_two(number): """ Check if a number is a power of two or not """ # This is a fast method to check for a power of two. # # A power of two has this structure: 100000 (one or more zeroes) # This is the same number minus one: 011111 (composed by ones) # This is the bitwise and: 000000 # # This is true only for every power of two return number != 0 and (number & (number-1)) == 0 barman-2.3/barman/version.py0000644000076500000240000000141013153300354016643 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . ''' This module contains the current Barman version. ''' __version__ = '2.3' barman-2.3/barman/wal_archiver.py0000644000076500000240000011233213137721573017646 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see import collections import datetime import errno import filecmp import logging import os import shutil from abc import ABCMeta, abstractmethod from glob import glob from distutils.version import LooseVersion as Version from barman import output, xlog from barman.command_wrappers import CommandFailedException, PgReceiveXlog from barman.exceptions import (AbortedRetryHookScript, ArchiverFailure, DuplicateWalFile, MatchingDuplicateWalFile) from barman.hooks import HookScriptRunner, RetryHookScriptRunner from barman.infofile import WalFileInfo from barman.remote_status import RemoteStatusMixin from barman.utils import fsync_dir, mkpath, with_metaclass _logger = logging.getLogger(__name__) class WalArchiverQueue(list): def __init__(self, items, errors=None, skip=None, batch_size=0): """ A WalArchiverQueue is a list of WalFileInfo which has two extra attribute list: * errors: containing a list of unrecognized files * skip: containing a list of skipped files. It also stores batch run size information in case it is requested by configuration, in order to limit the number of WAL files that are processed in a single run of the archive-wal command. :param items: iterable from which initialize the list :param batch_size: size of the current batch run (0=unlimited) :param errors: an optional list of unrecognized files :param skip: an optional list of skipped files """ super(WalArchiverQueue, self).__init__(items) self.skip = [] self.errors = [] if skip is not None: self.skip = skip if errors is not None: self.errors = errors # Normalises batch run size if batch_size > 0: self.batch_size = batch_size else: self.batch_size = 0 @property def size(self): """ Number of valid WAL segments waiting to be processed (in total) :return int: total number of valid WAL files """ return len(self) @property def run_size(self): """ Number of valid WAL files to be processed in this run - takes in consideration the batch size :return int: number of valid WAL files for this batch run """ # In case a batch size has been explicitly specified # (i.e. batch_size > 0), returns the minimum number between # batch size and the queue size. Otherwise, simply # returns the total queue size (unlimited batch size). if self.batch_size > 0: return min(self.size, self.batch_size) return self.size class WalArchiver(with_metaclass(ABCMeta, RemoteStatusMixin)): """ Base class for WAL archiver objects """ def __init__(self, backup_manager, name): """ Base class init method. :param backup_manager: The backup manager :param name: The name of this archiver :return: """ self.backup_manager = backup_manager self.server = backup_manager.server self.config = backup_manager.config self.name = name super(WalArchiver, self).__init__() def receive_wal(self, reset=False): """ Manage reception of WAL files. Does nothing by default. Some archiver classes, like the StreamingWalArchiver, have a full implementation. :param bool reset: When set, resets the status of receive-wal :raise ArchiverFailure: when something goes wrong """ def archive(self, verbose=True): """ Archive WAL files, discarding duplicates or those that are not valid. :param boolean verbose: Flag for verbose output """ compressor = self.backup_manager.compression_manager.get_compressor() stamp = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') processed = 0 header = "Processing xlog segments from %s for %s" % ( self.name, self.config.name) # Get the next batch of WAL files to be processed batch = self.get_next_batch() # Analyse the batch and properly log the information if batch.size: if batch.size > batch.run_size: # Batch mode enabled _logger.info("Found %s xlog segments from %s for %s." " Archive a batch of %s segments in this run.", batch.size, self.name, self.config.name, batch.run_size) header += " (batch size: %s)" % batch.run_size else: # Single run mode (traditional) _logger.info("Found %s xlog segments from %s for %s." " Archive all segments in one run.", batch.size, self.name, self.config.name) else: _logger.info("No xlog segments found from %s for %s.", self.name, self.config.name) # Print the header (verbose mode) if verbose: output.info(header, log=False) # Loop through all available WAL files for wal_info in batch: # Print the header (non verbose mode) if not processed and not verbose: output.info(header, log=False) # Exit when archive batch size is reached if processed >= batch.run_size: _logger.debug("Batch size reached (%s) - " "Exit %s process for %s", batch.batch_size, self.name, self.config.name) break processed += 1 # Report to the user the WAL file we are archiving output.info("\t%s", wal_info.name, log=False) _logger.info("Archiving segment %s of %s from %s: %s/%s", processed, batch.run_size, self.name, self.config.name, wal_info.name) # Archive the WAL file try: self.archive_wal(compressor, wal_info) except MatchingDuplicateWalFile: # We already have this file. Simply unlink the file. os.unlink(wal_info.orig_filename) continue except DuplicateWalFile: output.info("\tError: %s is already present in server %s. " "File moved to errors directory.", wal_info.name, self.config.name) error_dst = os.path.join( self.config.errors_directory, "%s.%s.duplicate" % (wal_info.name, stamp)) # TODO: cover corner case of duplication (unlikely, # but theoretically possible) shutil.move(wal_info.orig_filename, error_dst) continue except AbortedRetryHookScript as e: _logger.warning("Archiving of %s/%s aborted by " "pre_archive_retry_script." "Reason: %s" % (self.config.name, wal_info.name, e)) return if processed: _logger.debug("Archived %s out of %s xlog segments from %s for %s", processed, batch.size, self.name, self.config.name) elif verbose: output.info("\tno file found", log=False) if batch.errors: output.info("Some unknown objects have been found while " "processing xlog segments for %s. " "Objects moved to errors directory:", self.config.name, log=False) # Log unexpected files _logger.warning("Archiver is about to move %s unexpected file(s) " "to errors directory for %s from %s", len(batch.errors), self.config.name, self.name) for error in batch.errors: basename = os.path.basename(error) output.info("\t%s", basename, log=False) # Print informative log line. _logger.warning("Moving unexpected file for %s from %s: %s", self.config.name, self.name, basename) error_dst = os.path.join( self.config.errors_directory, "%s.%s.unknown" % (basename, stamp)) try: shutil.move(error, error_dst) except IOError as e: if e.errno == errno.ENOENT: _logger.warning('%s not found' % error) def archive_wal(self, compressor, wal_info): """ Archive a WAL segment and update the wal_info object :param compressor: the compressor for the file (if any) :param WalFileInfo wal_info: the WAL file is being processed """ src_file = wal_info.orig_filename src_dir = os.path.dirname(src_file) dst_file = wal_info.fullpath(self.server) tmp_file = dst_file + '.tmp' dst_dir = os.path.dirname(dst_file) error = None try: # Run the pre_archive_script if present. script = HookScriptRunner(self.backup_manager, 'archive_script', 'pre') script.env_from_wal_info(wal_info, src_file) script.run() # Run the pre_archive_retry_script if present. retry_script = RetryHookScriptRunner(self.backup_manager, 'archive_retry_script', 'pre') retry_script.env_from_wal_info(wal_info, src_file) retry_script.run() # Check if destination already exists if os.path.exists(dst_file): src_uncompressed = src_file dst_uncompressed = dst_file dst_info = WalFileInfo.from_file(dst_file) try: comp_manager = self.backup_manager.compression_manager if dst_info.compression is not None: dst_uncompressed = dst_file + '.uncompressed' comp_manager.get_compressor( compression=dst_info.compression).decompress( dst_file, dst_uncompressed) if wal_info.compression: src_uncompressed = src_file + '.uncompressed' comp_manager.get_compressor( compression=wal_info.compression).decompress( src_file, src_uncompressed) # Directly compare files. # When the files are identical # raise a MatchingDuplicateWalFile exception, # otherwise raise a DuplicateWalFile exception. if filecmp.cmp(dst_uncompressed, src_uncompressed): raise MatchingDuplicateWalFile(wal_info) else: raise DuplicateWalFile(wal_info) finally: if src_uncompressed != src_file: os.unlink(src_uncompressed) if dst_uncompressed != dst_file: os.unlink(dst_uncompressed) mkpath(dst_dir) # Compress the file only if not already compressed if compressor and not wal_info.compression: compressor.compress(src_file, tmp_file) # Perform the real filesystem operation with the xlogdb lock taken. # This makes the operation atomic from the xlogdb file POV with self.server.xlogdb('a') as fxlogdb: if compressor and not wal_info.compression: shutil.copystat(src_file, tmp_file) os.rename(tmp_file, dst_file) os.unlink(src_file) # Update wal_info stat = os.stat(dst_file) wal_info.size = stat.st_size wal_info.compression = compressor.compression else: # Try to atomically rename the file. If successful, # the renaming will be an atomic operation # (this is a POSIX requirement). try: os.rename(src_file, dst_file) except OSError: # Source and destination are probably on different # filesystems shutil.copy2(src_file, tmp_file) os.rename(tmp_file, dst_file) os.unlink(src_file) # At this point the original file has been removed wal_info.orig_filename = None # Execute fsync() on the archived WAL file file_fd = os.open(dst_file, os.O_RDONLY) os.fsync(file_fd) os.close(file_fd) # Execute fsync() on the archived WAL containing directory fsync_dir(dst_dir) # Execute fsync() also on the incoming directory fsync_dir(src_dir) # Updates the information of the WAL archive with # the latest segments fxlogdb.write(wal_info.to_xlogdb_line()) # flush and fsync for every line fxlogdb.flush() os.fsync(fxlogdb.fileno()) except Exception as e: # In case of failure save the exception for the post scripts error = e raise # Ensure the execution of the post_archive_retry_script and # the post_archive_script finally: # Run the post_archive_retry_script if present. try: retry_script = RetryHookScriptRunner(self, 'archive_retry_script', 'post') retry_script.env_from_wal_info(wal_info, dst_file, error) retry_script.run() except AbortedRetryHookScript as e: # Ignore the ABORT_STOP as it is a post-hook operation _logger.warning("Ignoring stop request after receiving " "abort (exit code %d) from post-archive " "retry hook script: %s", e.hook.exit_status, e.hook.script) # Run the post_archive_script if present. script = HookScriptRunner(self, 'archive_script', 'post', error) script.env_from_wal_info(wal_info, dst_file) script.run() @abstractmethod def get_next_batch(self): """ Return a WalArchiverQueue containing the WAL files to be archived. :rtype: WalArchiverQueue """ @abstractmethod def check(self, check_strategy): """ Perform specific checks for the archiver - invoked by server.check_postgres :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ @abstractmethod def status(self): """ Set additional status info - invoked by Server.status() """ @staticmethod def summarise_error_files(error_files): """ Summarise a error files list :param list[str] error_files: Error files list to summarise :return str: A summary, None if there are no error files """ if not error_files: return None # The default value for this dictionary will be 0 counters = collections.defaultdict(int) # Count the file types for name in error_files: if name.endswith(".error"): counters['not relevant'] += 1 elif name.endswith(".duplicate"): counters['duplicates'] += 1 elif name.endswith(".unknown"): counters['unknown'] += 1 else: counters['unknown failure'] += 1 # Return a summary list of the form: "item a: 2, item b: 5" return ', '.join("%s: %s" % entry for entry in counters.items()) class FileWalArchiver(WalArchiver): """ Manager of file-based WAL archiving operations (aka 'log shipping'). """ def __init__(self, backup_manager): super(FileWalArchiver, self).__init__(backup_manager, 'file archival') def fetch_remote_status(self): """ Returns the status of the FileWalArchiver. This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ result = dict.fromkeys( ['archive_mode', 'archive_command'], None) postgres = self.server.postgres # If Postgres is not available we cannot detect anything if not postgres: return result # Query the database for 'archive_mode' and 'archive_command' result['archive_mode'] = postgres.get_setting('archive_mode') result['archive_command'] = postgres.get_setting('archive_command') # Add pg_stat_archiver statistics if the view is supported pg_stat_archiver = postgres.get_archiver_stats() if pg_stat_archiver is not None: result.update(pg_stat_archiver) return result def get_next_batch(self): """ Returns the next batch of WAL files that have been archived through a PostgreSQL's 'archive_command' (in the 'incoming' directory) :return: WalArchiverQueue: list of WAL files """ # Get the batch size from configuration (0 = unlimited) batch_size = self.config.archiver_batch_size # List and sort all files in the incoming directory file_names = glob(os.path.join( self.config.incoming_wals_directory, '*')) file_names.sort() # Process anything that looks like a valid WAL file. Anything # else is treated like an error/anomaly files = [] errors = [] for file_name in file_names: # Ignore temporary files if file_name.endswith('.tmp'): continue if xlog.is_any_xlog_file(file_name) and os.path.isfile(file_name): files.append(file_name) else: errors.append(file_name) # Build the list of WalFileInfo wal_files = [WalFileInfo.from_file(f) for f in files] return WalArchiverQueue(wal_files, batch_size=batch_size, errors=errors) def check(self, check_strategy): """ Perform additional checks for FileWalArchiver - invoked by server.check_postgres :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('archive_mode') remote_status = self.get_remote_status() # If archive_mode is None, there are issues connecting to PostgreSQL if remote_status['archive_mode'] is None: return # Check archive_mode parameter: must be on if remote_status['archive_mode'] in ('on', 'always'): check_strategy.result(self.config.name, True) else: msg = "please set it to 'on'" if self.server.postgres.server_version >= 90500: msg += " or 'always'" check_strategy.result(self.config.name, False, hint=msg) check_strategy.init_check('archive_command') if remote_status['archive_command'] and \ remote_status['archive_command'] != '(disabled)': check_strategy.result(self.config.name, True, check='archive_command') # Report if the archiving process works without issues. # Skip if the archive_command check fails # It can be None if PostgreSQL is older than 9.4 if remote_status.get('is_archiving') is not None: check_strategy.result( self.config.name, remote_status['is_archiving'], check='continuous archiving') else: check_strategy.result( self.config.name, False, hint='please set it accordingly to documentation') def status(self): """ Set additional status info - invoked by Server.status() """ remote_status = self.get_remote_status() # If archive_mode is None, there are issues connecting to PostgreSQL if remote_status['archive_mode'] is None: return output.result('status', self.config.name, "archive_command", "PostgreSQL 'archive_command' setting", remote_status['archive_command'] or "FAILED (please set it accordingly to documentation)") last_wal = remote_status.get('last_archived_wal') # If PostgreSQL is >= 9.4 we have the last_archived_time if last_wal and remote_status.get('last_archived_time'): last_wal += ", at %s" % ( remote_status['last_archived_time'].ctime()) output.result('status', self.config.name, "last_archived_wal", "Last archived WAL", last_wal or "No WAL segment shipped yet") # Set output for WAL archive failures (PostgreSQL >= 9.4) if remote_status.get('failed_count') is not None: remote_fail = str(remote_status['failed_count']) if int(remote_status['failed_count']) > 0: remote_fail += " (%s at %s)" % ( remote_status['last_failed_wal'], remote_status['last_failed_time'].ctime()) output.result('status', self.config.name, 'failed_count', 'Failures of WAL archiver', remote_fail) # Add hourly archive rate if available (PostgreSQL >= 9.4) and > 0 if remote_status.get('current_archived_wals_per_second'): output.result( 'status', self.config.name, 'server_archived_wals_per_hour', 'Server WAL archiving rate', '%0.2f/hour' % ( 3600 * remote_status['current_archived_wals_per_second'])) class StreamingWalArchiver(WalArchiver): """ Object used for the management of streaming WAL archive operation. """ def __init__(self, backup_manager): super(StreamingWalArchiver, self).__init__(backup_manager, 'streaming') def fetch_remote_status(self): """ Execute checks for replication-based wal archiving This method does not raise any exception in case of errors, but set the missing values to None in the resulting dictionary. :rtype: dict[str, None|str] """ remote_status = dict.fromkeys( ('pg_receivexlog_compatible', 'pg_receivexlog_installed', 'pg_receivexlog_path', 'pg_receivexlog_supports_slots', 'pg_receivexlog_synchronous', 'pg_receivexlog_version'), None) # Test pg_receivexlog existence version_info = PgReceiveXlog.get_version_info( self.server.path) if version_info['full_path']: remote_status["pg_receivexlog_installed"] = True remote_status["pg_receivexlog_path"] = version_info['full_path'] remote_status["pg_receivexlog_version"] = ( version_info['full_version']) pgreceivexlog_version = version_info['major_version'] else: remote_status["pg_receivexlog_installed"] = False return remote_status # Retrieve the PostgreSQL version pg_version = None if self.server.streaming is not None: pg_version = self.server.streaming.server_major_version # If one of the version is unknown we cannot compare them if pgreceivexlog_version is None or pg_version is None: return remote_status # pg_version is not None so transform into a Version object # for easier comparison between versions pg_version = Version(pg_version) # Set conservative default values (False) for modern features remote_status["pg_receivexlog_compatible"] = False remote_status['pg_receivexlog_supports_slots'] = False remote_status["pg_receivexlog_synchronous"] = False # pg_receivexlog 9.2 is compatible only with PostgreSQL 9.2. if "9.2" == pg_version == pgreceivexlog_version: remote_status["pg_receivexlog_compatible"] = True # other versions are compatible with lesser versions of PostgreSQL # WARNING: The development versions of `pg_receivexlog` are considered # higher than the stable versions here, but this is not an issue # because it accepts everything that is less than # the `pg_receivexlog` version(e.g. '9.6' is less than '9.6devel') elif "9.2" < pg_version <= pgreceivexlog_version: # At least PostgreSQL 9.3 is required here remote_status["pg_receivexlog_compatible"] = True # replication slots are supported starting from version 9.4 if "9.4" <= pg_version <= pgreceivexlog_version: remote_status['pg_receivexlog_supports_slots'] = True # Synchronous WAL streaming requires replication slots # and pg_receivexlog >= 9.5 if "9.4" <= pg_version and "9.5" <= pgreceivexlog_version: remote_status["pg_receivexlog_synchronous"] = ( self._is_synchronous()) return remote_status def receive_wal(self, reset=False): """ Creates a PgReceiveXlog object and issues the pg_receivexlog command for a specific server :param bool reset: When set reset the status of receive-wal :raise ArchiverFailure: when something goes wrong """ # Ensure the presence of the destination directory mkpath(self.config.streaming_wals_directory) # Check if is a reset request if reset: self._reset_streaming_status() return # Execute basic sanity checks on PostgreSQL connection streaming_status = self.server.streaming.get_remote_status() if streaming_status["streaming_supported"] is None: raise ArchiverFailure( 'failed opening the PostgreSQL streaming connection ' 'for server %s' % (self.config.name)) elif not streaming_status["streaming_supported"]: raise ArchiverFailure( 'PostgreSQL version too old (%s < 9.2)' % self.server.streaming.server_txt_version) # Execute basic sanity checks on pg_receivexlog remote_status = self.get_remote_status() if not remote_status["pg_receivexlog_installed"]: raise ArchiverFailure( 'pg_receivexlog not present in $PATH') if not remote_status['pg_receivexlog_compatible']: raise ArchiverFailure( 'pg_receivexlog version not compatible with ' 'PostgreSQL server version') # Execute sanity check on replication slot usage if self.config.slot_name: # Check if slots are supported if not remote_status['pg_receivexlog_supports_slots']: raise ArchiverFailure( 'Physical replication slot not supported by %s ' '(9.4 or higher is required)' % self.server.streaming.server_txt_version) # Check if the required slot exists postgres_status = self.server.postgres.get_remote_status() if postgres_status['replication_slot'] is None: raise ArchiverFailure( "replication slot '%s' doesn't exist. " "Please execute " "'barman receive-wal --create-slot %s'" % (self.config.slot_name, self.config.name)) # Check if the required slot is available if postgres_status['replication_slot'].active: raise ArchiverFailure( "replication slot '%s' is already in use" % (self.config.slot_name,)) # Make sure we are not wasting precious PostgreSQL resources self.server.close() _logger.info('Activating WAL archiving through streaming protocol') try: output_handler = PgReceiveXlog.make_output_handler( self.config.name + ': ') receive = PgReceiveXlog( connection=self.server.streaming, destination=self.config.streaming_wals_directory, command=remote_status['pg_receivexlog_path'], version=remote_status['pg_receivexlog_version'], app_name=self.config.streaming_archiver_name, path=self.server.path, slot_name=self.config.slot_name, synchronous=remote_status['pg_receivexlog_synchronous'], out_handler=output_handler, err_handler=output_handler ) # Finally execute the pg_receivexlog process receive.execute() except CommandFailedException as e: # Retrieve the return code from the exception ret_code = e.args[0]['ret'] if ret_code < 0: # If the return code is negative, then pg_receivexlog # was terminated by a signal msg = "pg_receivexlog terminated by signal: %s" \ % abs(ret_code) else: # Otherwise terminated with an error msg = "pg_receivexlog terminated with error code: %s"\ % ret_code raise ArchiverFailure(msg) except KeyboardInterrupt: # This is a normal termination, so there is nothing to do beside # informing the user. output.info('SIGINT received. Terminate gracefully.') def _reset_streaming_status(self): """ Reset the status of receive-wal removing any .partial files """ output.info("Resetting receive-wal directory status") partial_files = glob(os.path.join( self.config.streaming_wals_directory, '*.partial')) for partial in partial_files: output.info("Removing status file %s" % partial) os.unlink(partial) def get_next_batch(self): """ Returns the next batch of WAL files that have been archived via streaming replication (in the 'streaming' directory) This method always leaves one file in the "streaming" directory, because the 'pg_receivexlog' process needs at least one file to detect the current streaming position after a restart. :return: WalArchiverQueue: list of WAL files """ # Get the batch size from configuration (0 = unlimited) batch_size = self.config.streaming_archiver_batch_size # List and sort all files in the incoming directory file_names = glob(os.path.join( self.config.streaming_wals_directory, '*')) file_names.sort() # Process anything that looks like a valid WAL file, # including partial ones and history files. # Anything else is treated like an error/anomaly files = [] skip = [] errors = [] for file_name in file_names: # Ignore temporary files if file_name.endswith('.tmp'): continue if not os.path.isfile(file_name): errors.append(file_name) elif xlog.is_partial_file(file_name): skip.append(file_name) elif xlog.is_any_xlog_file(file_name): files.append(file_name) else: errors.append(file_name) # In case of more than a partial file, keep the last # and treat the rest as normal files if len(skip) > 1: partials = skip[:-1] _logger.info('Archiving partial files for server %s: %s' % (self.config.name, ", ".join([os.path.basename(f) for f in partials]))) files.extend(partials) skip = skip[-1:] # Keep the last full WAL file in case no partial file is present elif len(skip) == 0 and files: skip.append(files.pop()) # Build the list of WalFileInfo wal_files = [WalFileInfo.from_file(f, compression=None) for f in files] return WalArchiverQueue(wal_files, batch_size=batch_size, errors=errors, skip=skip) def check(self, check_strategy): """ Perform additional checks for StreamingWalArchiver - invoked by server.check_postgres :param CheckStrategy check_strategy: the strategy for the management of the results of the various checks """ check_strategy.init_check('pg_receivexlog') # Check the version of pg_receivexlog remote_status = self.get_remote_status() check_strategy.result( self.config.name, remote_status['pg_receivexlog_installed']) hint = None check_strategy.init_check('pg_receivexlog compatible') if not remote_status['pg_receivexlog_compatible']: pg_version = 'Unknown' if self.server.streaming is not None: pg_version = self.server.streaming.server_txt_version hint = "PostgreSQL version: %s, pg_receivexlog version: %s" % ( pg_version, remote_status['pg_receivexlog_version'] ) check_strategy.result(self.config.name, remote_status['pg_receivexlog_compatible'], hint=hint) # Check if pg_receivexlog is running, by retrieving a list # of running 'receive-wal' processes from the process manager. receiver_list = self.server.process_manager.list('receive-wal') # If there's at least one 'receive-wal' process running for this # server, the test is passed check_strategy.init_check('receive-wal running') if receiver_list: check_strategy.result( self.config.name, True) else: check_strategy.result( self.config.name, False, hint='See the Barman log file for more details') def _is_synchronous(self): """ Check if receive-wal process is eligible for synchronous replication The receive-wal process is eligible for synchronous replication if `synchronous_standby_names` is configured and contains the value of `streaming_archiver_name` :rtype: bool """ # Nothing to do if postgres connection is not working postgres = self.server.postgres if postgres is None or postgres.server_txt_version is None: return None # Check if synchronous WAL streaming can be enabled # by peeking 'synchronous_standby_names' postgres_status = postgres.get_remote_status() syncnames = postgres_status['synchronous_standby_names'] _logger.debug("Look for '%s' in " "'synchronous_standby_names': %s", self.config.streaming_archiver_name, syncnames) # The receive-wal process is eligible for synchronous replication # if `synchronous_standby_names` is configured and contains # the value of `streaming_archiver_name` synchronous = (syncnames and self.config.streaming_archiver_name in syncnames) _logger.info('Synchronous WAL streaming for %s: %s', self.config.streaming_archiver_name, synchronous) return synchronous def status(self): """ Set additional status info - invoked by Server.status() """ # TODO: Add status information for WAL streaming barman-2.3/barman/xlog.py0000644000076500000240000003375613134472625016163 0ustar mnenciastaff00000000000000# Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . """ This module contains functions to retrieve information about xlog files """ import collections import os import re from tempfile import NamedTemporaryFile from barman.exceptions import BadHistoryFileContents, BadXlogSegmentName # xlog file segment name parser (regular expression) _xlog_re = re.compile(r''' ^ ([\dA-Fa-f]{8}) # everything has a timeline (?: ([\dA-Fa-f]{8})([\dA-Fa-f]{8}) # segment name, if a wal file (?: # and optional \.[\dA-Fa-f]{8}\.backup # offset, if a backup label | \.partial # partial, if a partial file )? | \.history # or only .history, if a history file ) $ ''', re.VERBOSE) # xlog location parser for concurrent backup (regular expression) _location_re = re.compile(r'^([\dA-F]+)/([\dA-F]+)$') # Taken from xlog_internal.h from PostgreSQL sources #: XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2 #: and larger than XLOG_BLCKSZ (preferably, a great deal larger than #: XLOG_BLCKSZ). DEFAULT_XLOG_SEG_SIZE = 1 << 24 #: This namedtuple is a container for the information #: contained inside history files HistoryFileData = collections.namedtuple( 'HistoryFileData', 'tli parent_tli switchpoint reason') def is_any_xlog_file(path): """ Return True if the xlog is either a WAL segment, a .backup file or a .history file, False otherwise. It supports either a full file path or a simple file name. :param str path: the file name to test :rtype: bool """ match = _xlog_re.match(os.path.basename(path)) if match: return True return False def is_history_file(path): """ Return True if the xlog is a .history file, False otherwise It supports either a full file path or a simple file name. :param str path: the file name to test :rtype: bool """ match = _xlog_re.search(os.path.basename(path)) if match and match.group(0).endswith('.history'): return True return False def is_backup_file(path): """ Return True if the xlog is a .backup file, False otherwise It supports either a full file path or a simple file name. :param str path: the file name to test :rtype: bool """ match = _xlog_re.search(os.path.basename(path)) if match and match.group(0).endswith('.backup'): return True return False def is_partial_file(path): """ Return True if the xlog is a .partial file, False otherwise It supports either a full file path or a simple file name. :param str path: the file name to test :rtype: bool """ match = _xlog_re.search(os.path.basename(path)) if match and match.group(0).endswith('.partial'): return True return False def is_wal_file(path): """ Return True if the xlog is a regular xlog file, False otherwise It supports either a full file path or a simple file name. :param str path: the file name to test :rtype: bool """ match = _xlog_re.search(os.path.basename(path)) if (match and not match.group(0).endswith('.backup') and not match.group(0).endswith('.history') and not match.group(0).endswith('.partial')): return True return False def decode_segment_name(path): """ Retrieve the timeline, log ID and segment ID from the name of a xlog segment It can handle either a full file path or a simple file name. :param str path: the file name to decode :rtype: list[int] """ name = os.path.basename(path) match = _xlog_re.match(name) if not match: raise BadXlogSegmentName(name) return [int(x, 16) if x else None for x in match.groups()] def encode_segment_name(tli, log, seg): """ Build the xlog segment name based on timeline, log ID and segment ID :param int tli: timeline number :param int log: log number :param int seg: segment number :return str: segment file name """ return "%08X%08X%08X" % (tli, log, seg) def encode_history_file_name(tli): """ Build the history file name based on timeline :return str: history file name """ return "%08X.history" % (tli,) def xlog_segments_per_file(xlog_segment_size): """ Given that WAL files are named using the following pattern: this is the number of XLOG segments in an XLOG file. By XLOG file we don't mean an actual file on the filesystem, but the definition used in the PostgreSQL sources: meaning a set of files containing the same file number. :param int xlog_segment_size: The XLOG segment size in bytes :return int: The number of segments in an XLOG file """ return 0xffffffff // xlog_segment_size def xlog_file_size(xlog_segment_size): """ Given that WAL files are named using the following pattern: this is the size in bytes of an XLOG file, which is composed on many segments. See the documentation of `xlog_segments_per_file` for a commentary on the definition of `XLOG` file. :param int xlog_segment_size: The XLOG segment size in bytes :return int: The size of an XLOG file """ return xlog_segment_size * xlog_segments_per_file(xlog_segment_size) def generate_segment_names(begin, end=None, version=None, xlog_segment_size=None): """ Generate a sequence of XLOG segments starting from ``begin`` If an ``end`` segment is provided the sequence will terminate after returning it, otherwise the sequence will never terminate. If the XLOG segment size is known, this generator is precise, switching to the next file when required. It the XLOG segment size is unknown, this generator will generate all the possible XLOG file names. The size of an XLOG segment can be every power of 2 between the XLOG block size (8Kib) and the size of a log segment (4Gib) :param str begin: begin segment name :param str|None end: optional end segment name :param int|None version: optional postgres version as an integer (e.g. 90301 for 9.3.1) :param int xlog_segment_size: the size of a XLOG segment :rtype: collections.Iterable[str] :raise: BadXlogSegmentName """ begin_tli, begin_log, begin_seg = decode_segment_name(begin) end_tli, end_log, end_seg = None, None, None if end: end_tli, end_log, end_seg = decode_segment_name(end) # this method doesn't support timeline changes assert begin_tli == end_tli, ( "Begin segment (%s) and end segment (%s) " "must have the same timeline part" % (begin, end)) # If version is less than 9.3 the last segment must be skipped skip_last_segment = version is not None and version < 90300 # This is the number of XLOG segments in an XLOG file. By XLOG file # we don't mean an actual file on the filesystem, but the definition # used in the PostgreSQL sources: a set of files containing the # same file number. if xlog_segment_size: # The generator is operating is precise and correct mode: # knowing exactly when a switch to the next file is required xlog_seg_per_file = xlog_segments_per_file(xlog_segment_size) else: # The generator is operating only in precise mode: generating every # possible XLOG file name. xlog_seg_per_file = 0x7ffff # Start from the first xlog and generate the segments sequentially # If ``end`` has been provided, the while condition ensure the termination # otherwise this generator will never stop cur_log, cur_seg = begin_log, begin_seg while end is None or \ cur_log < end_log or \ (cur_log == end_log and cur_seg <= end_seg): yield encode_segment_name(begin_tli, cur_log, cur_seg) cur_seg += 1 if cur_seg > xlog_seg_per_file or ( skip_last_segment and cur_seg == xlog_seg_per_file): cur_seg = 0 cur_log += 1 def hash_dir(path): """ Get the directory where the xlog segment will be stored It can handle either a full file path or a simple file name. :param str|unicode path: xlog file name :return str: directory name """ tli, log, _ = decode_segment_name(path) # tli is always not None if log is not None: return "%08X%08X" % (tli, log) else: return '' def parse_lsn(lsn_string): """ Transform a string XLOG location, formatted as %X/%X, in the corresponding numeric representation :param str lsn_string: the string XLOG location, i.e. '2/82000168' :rtype: int """ lsn_list = lsn_string.split('/') if len(lsn_list) != 2: raise ValueError('Invalid LSN: %s', lsn_string) return (int(lsn_list[0], 16) << 32) + int(lsn_list[1], 16) def diff_lsn(lsn_string1, lsn_string2): """ Calculate the difference in bytes between two string XLOG location, formatted as %X/%X Tis function is a Python implementation of the ``pg_xlog_location_diff(str, str)`` PostgreSQL function. :param str lsn_string1: the string XLOG location, i.e. '2/82000168' :param str lsn_string2: the string XLOG location, i.e. '2/82000168' :rtype: int """ # If one the input is None returns None if lsn_string1 is None or lsn_string2 is None: return None return parse_lsn(lsn_string1) - parse_lsn(lsn_string2) def format_lsn(lsn): """ Transform a numeric XLOG location, in the corresponding %X/%X string representation :param int lsn: numeric XLOG location :rtype: str """ return "%X/%X" % (lsn >> 32, lsn & 0xFFFFFFFF) def location_to_xlogfile_name_offset(location, timeline, xlog_segment_size): """ Convert transaction log location string to file_name and file_offset This is a reimplementation of pg_xlogfile_name_offset PostgreSQL function This method returns a dictionary containing the following data: * file_name * file_offset :param str location: XLOG location :param int timeline: timeline :param int xlog_segment_size: the size of a XLOG segment :rtype: dict """ lsn = parse_lsn(location) log = lsn >> 32 seg = (lsn & xlog_file_size(xlog_segment_size)) >> 24 offset = lsn & 0xFFFFFF return { 'file_name': encode_segment_name(timeline, log, seg), 'file_offset': offset, } def location_from_xlogfile_name_offset(file_name, file_offset): """ Convert file_name and file_offset to a transaction log location. This is the inverted function of PostgreSQL's pg_xlogfile_name_offset function. :param str file_name: a WAL file name :param int file_offset: a numeric offset :rtype: str """ decoded_segment = decode_segment_name(file_name) location = ((decoded_segment[1] << 32) + (decoded_segment[2] << 24) + file_offset) return format_lsn(location) def decode_history_file(wal_info, comp_manager): """ Read an history file and parse its contents. Each line in the file represents a timeline switch, each field is separated by tab, empty lines are ignored and lines starting with '#' are comments. Each line is composed by three fields: parentTLI, switchpoint and reason. "parentTLI" is the ID of the parent timeline. "switchpoint" is the WAL position where the switch happened "reason" is an human-readable explanation of why the timeline was changed The method requires a CompressionManager object to handle the eventual compression of the history file. :param barman.infofile.WalFileInfo wal_info: history file obj :param comp_manager: compression manager used in case of history file compression :return List[HistoryFileData]: information from the history file """ path = wal_info.orig_filename # Decompress the file if needed if wal_info.compression: # Use a NamedTemporaryFile to avoid explicit cleanup uncompressed_file = NamedTemporaryFile( dir=os.path.dirname(path), prefix='.%s.' % wal_info.name, suffix='.uncompressed') path = uncompressed_file.name comp_manager.get_compressor(wal_info.compression).decompress( wal_info.orig_filename, path) # Extract the timeline from history file name tli, _, _ = decode_segment_name(wal_info.name) lines = [] with open(path) as fp: for line in fp: line = line.strip() # Skip comments and empty lines if line.startswith("#"): continue # Skip comments and empty lines if len(line) == 0: continue # Use tab as separator contents = line.split('\t') if len(contents) != 3: # Invalid content of the line raise BadHistoryFileContents(path) history = HistoryFileData( tli=tli, parent_tli=int(contents[0]), switchpoint=parse_lsn(contents[1]), reason=contents[2]) lines.append(history) # Empty history file or containing invalid content if len(lines) == 0: raise BadHistoryFileContents(path) else: return lines barman-2.3/barman.egg-info/0000755000076500000240000000000013153300776016312 5ustar mnenciastaff00000000000000barman-2.3/barman.egg-info/dependency_links.txt0000644000076500000240000000000113153300776022360 0ustar mnenciastaff00000000000000 barman-2.3/barman.egg-info/PKG-INFO0000644000076500000240000000256413153300776017416 0ustar mnenciastaff00000000000000Metadata-Version: 1.1 Name: barman Version: 2.3 Summary: Backup and Recovery Manager for PostgreSQL Home-page: http://www.pgbarman.org/ Author: 2ndQuadrant Limited Author-email: info@2ndquadrant.com License: GPL-3.0 Description: Barman (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. Barman is distributed under GNU GPL 3 and maintained by 2ndQuadrant. Platform: Linux Platform: Mac OS X Classifier: Environment :: Console Classifier: Development Status :: 5 - Production/Stable Classifier: Topic :: System :: Archiving :: Backup Classifier: Topic :: Database Classifier: Topic :: System :: Recovery Tools Classifier: Intended Audience :: System Administrators Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 barman-2.3/barman.egg-info/requires.txt0000644000076500000240000000010213153300776020703 0ustar mnenciastaff00000000000000psycopg2>=2.4.2 argh<=0.26.2,>=0.21.2 python-dateutil argcomplete barman-2.3/barman.egg-info/SOURCES.txt0000644000076500000240000000376713153300776020213 0ustar mnenciastaff00000000000000AUTHORS ChangeLog INSTALL LICENSE MANIFEST.in NEWS README.rst setup.cfg setup.py barman/__init__.py barman/backup.py barman/backup_executor.py barman/cli.py barman/command_wrappers.py barman/compression.py barman/config.py barman/copy_controller.py barman/diagnose.py barman/exceptions.py barman/fs.py barman/hooks.py barman/infofile.py barman/lockfile.py barman/output.py barman/postgres.py barman/process.py barman/recovery_executor.py barman/remote_status.py barman/retention_policies.py barman/server.py barman/utils.py barman/version.py barman/wal_archiver.py barman/xlog.py barman.egg-info/PKG-INFO barman.egg-info/SOURCES.txt barman.egg-info/dependency_links.txt barman.egg-info/requires.txt barman.egg-info/top_level.txt bin/barman doc/Makefile doc/barman.1 doc/barman.1.md doc/barman.5 doc/barman.5.md doc/barman.conf doc/barman.d/ssh-server.conf-template doc/barman.d/streaming-server.conf-template doc/manual/00-head.en.md doc/manual/01-intro.en.md doc/manual/02-before_you_start.en.md doc/manual/10-design.en.md doc/manual/15-system_requirements.en.md doc/manual/16-installation.en.md doc/manual/17-configuration.en.md doc/manual/20-server_setup.en.md doc/manual/21-preliminary_steps.en.md doc/manual/22-config_file.en.md doc/manual/23-wal_streaming.en.md doc/manual/24-wal_archiving.en.md doc/manual/25-streaming_backup.en.md doc/manual/26-rsync_backup.en.md doc/manual/27-windows-support.en.md doc/manual/41-global-commands.en.md doc/manual/42-server-commands.en.md doc/manual/43-backup-commands.en.md doc/manual/50-feature-details.en.md doc/manual/65-troubleshooting.en.md doc/manual/66-about.en.md doc/manual/70-feature-matrix.en.md doc/manual/99-references.en.md rpm/barman.spec rpm/rhel5/python-dateutil-1.4.1-remove-embedded-timezone-data.patch rpm/rhel5/python26-argcomplete.spec rpm/rhel5/python26-argh.spec rpm/rhel5/python26-dateutil.spec rpm/rhel5/python26-psycopg2.spec rpm/rhel5/setup.cfg.patch rpm/rhel6/python-argcomplete.spec rpm/rhel6/python-argh.spec rpm/rhel7/python-argh.spec scripts/barman.bash_completionbarman-2.3/barman.egg-info/top_level.txt0000644000076500000240000000000713153300776021041 0ustar mnenciastaff00000000000000barman barman-2.3/bin/0000755000076500000240000000000013153300776014130 5ustar mnenciastaff00000000000000barman-2.3/bin/barman0000755000076500000240000000152313134472625015321 0ustar mnenciastaff00000000000000#!/usr/bin/env python # # Copyright (C) 2011-2017 2ndQuadrant Limited # # This file is part of Barman. # # Barman is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # Barman is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Barman. If not, see . # # PYTHON_ARGCOMPLETE_OK from barman.cli import main if __name__ == '__main__': main() else: raise NotImplementedError barman-2.3/ChangeLog0000644000076500000240000040765413153300354015142 0ustar mnenciastaff000000000000002017-09-04 Marco Nenciarini Update the ChangeLog file Set version to 2.3 2017-09-01 Marco Nenciarini Support `path_prefix` for PostgreSQL 10 With PostgreSQL 10, `pg_receivexlog` has been renamed to `pg_receivewal`. Barman will try both and will pick `pg_receivewal` in case it find both of them. The user can override this behavior by using the `path_prefix` setting and prepending a directory containing only the desired command version. 2017-08-30 Gabriele Bartolini Improve naming consistency with PostgreSQL 10 The `switch-xlog` command has been renamed to `switch-wal`. In commands output, the `xlog` word has been changed to `WAL` and `location` has been changed to `LSN` when appropriate. Improve documentation about recovery target 2017-08-29 Gabriele Bartolini Fix typo in doc for createuser 2017-08-26 Gabriele Bartolini Add `--target-immediate` option to `recover` command Throught the `--target-immediate` option, Barman writes the `recovery_target = 'immediate'` line to recovery.conf. By doing so, PostgreSQL will exit from recovery mode as soon as a consistent state is reached (end of the base backup). Only available for PostgreSQL 9.4 or newer. 2017-08-25 Gabriele Bartolini Show cluster state with `barman status` Similarly to `pg_controldata`, the `status` command now displays the cluster state of a PostgreSQL server ('in production' or 'in archive recovery'). 2017-08-23 Marco Nenciarini Support PostgreSQL 10 in `barman replication-status` command Invoke pg_basebackup with --no-slot option in PostgreSQL 10+ Starting from PostgreSQL 10 pg_basebackup uses a temporary replication slot unless explicitly instructed. Barman don't need it because it already stores the full WAL stream 2017-08-02 Marco Nenciarini Fix exception during exclusive backup with PostgreSQL 10 2017-08-01 Gabriele Bartolini Add `--network-compression` option to remote recovery Enable/disable network compression at run-time to remote recovery with `--network-compression` and `--no-network-compression` options. 2017-08-01 Marco Nenciarini Describe the actual error when invoking archiving hooks script In case of error the `BARMAN_ERROR` variable will contain a string starting with the name of the actual WALFileException subclass. 2017-07-28 Giulio Calacoci Ignore `.tmp` files in `incoming` and `streaming` directories The pg_receivewal utility occasionally writes to a `.tmp` file and then renames it to the final name to ensure an atomic creation. Barman must ignore this temporary file. Other programs could do the same, so Barman will ignore any file name with a `.tmp` suffix in the incoming and streaming directories. 2017-07-20 Leonardo Cecchi Add support to PostgreSQL 10 PostgreSQL 10 renames SQL functions, tools and options that references `xlog` to `wal`. Also WAL related functions referencing `location` have been renamed to `lsn`. That change have been made to improve consistency of terminology. This patch improves Barman adding support to PostgreSQL 10. 2017-07-26 Marco Nenciarini Make flake8 happy again Improve `barman diagnose` robustness 2017-07-22 Marco Nenciarini Fix high memory usage with parallel_jobs > 1 This patch avoids bloating the memory with objects duplicated by pickle/unpickle cycles. Fixes #116 2017-07-21 Marco Nenciarini Improve travis integration Fix unit test for FileWalArchiver to work with Python 3.6 Python 3.6 is stricter about what the `os.path.join` function can receive as input. This change has highlighted missing prerequisite setting in the FileWalArchiver.archive test. Set version 2.3a1 Fix exception in `barman get-wal --peek` requiring history files Fix exception during check if ssh works and conninfo is invalid Make sure that parallel workers are cleaned up in case of errors In CopyController code, at the end of parallel copy, it is necessary to terminate all the parallel workers, whether the copy finishes naturally or it is terminated by an error. Closes: #114 2017-07-17 Marco Nenciarini Update the ChangeLog file 2017-07-14 Marco Nenciarini Set version to 2.2 Update copyright 2017-07-12 Marco Nenciarini Log get-wal source host when called through SSH 2017-07-06 Leonardo Cecchi Check the number of files in the archiving queue Add a new configuration parameter, named `max_incoming_wals_queue`, that enables a new check for the number of files in the incoming directories. This check will fail if the number of WALs to be archived goes over the allowed threshold. This check does not block normal operations. The purpose of this check is to improve robustness of the Barman system when integrated in an alerting/monitoring infrastructure, by warning users in case Barman is not archiving fast enough. 2017-07-10 Marco Nenciarini Fix missing mock in test_pg_stat_archiver_status 2017-07-11 Leonardo Cecchi Show-backup fails on incomplete backups The barman `show-backup` command was failing on failed backups with a stacktrace. Fixed unit tests as well as the bug. 2017-07-05 Marco Nenciarini Update copy_controller unit tests 2017-07-04 Gabriele Bartolini Improve copy output messages 2017-07-03 Leonardo Cecchi Support dynamic WAL size Barman now asks PostgreSQL for the current WAL file size at every start of backup. This information is stored in the backup info and then used while showing statistics. Previously the size of WAL files was hardcoded into Barman, giving wrong statistics when PostgreSQL was compiled with a different value. IMPORTANT: Release notes must mention that the format of the backup info file has changed. A backup made with Barman 2.2 cannot be read by a previous version of Barman (in the rare event that a user downgrades version). 2017-07-05 Leonardo Cecchi Make cuncurrent backup killable Barman was able to recover from a KeyboardInterrupt while executing a backup, so you were not able to interrupt a backup anymore. This patch disable the handling of the SIGINT signal from the worker processes. The signal is now handled by the parent process, which is responsible for the termination of the children processes. The backup will be then stopped in the usual way. 2017-07-04 Gabriele Bartolini Allow replication-status to work against a standby 2017-07-04 Marco Nenciarini Close all PostgreSQL connections before starting pg_basebackup With this commit we avoid keeping an idle transaction during the file copy and fix incorrect end time reported at the end of the backup. Closes: #108, #104 Fix exception in show-backup with postgres backup_method 2017-06-01 Giulio Calacoci Safely handle paths containing special characters This patch makes Barman correctly handle path containing those characters having special meaning to the Unix shells such as quotes, double quotes, spaces, and so on. 2017-06-12 Gabriele Bartolini Improve backup completed message 2017-06-05 Marco Nenciarini Display statistics about the copy gathered during backup The statistics will be displayed in `barman show-backup` output as 'Copy time' and 'Estimated throughput' values. Fix an error message interpolation Make the method rsync_factory private The method rsync_factory should be private, as it must be used only from the copy controller. 2017-05-29 Giulio Calacoci Ensure incoming directory is empty when `archiver=off` This commit adds a check that warns the user when there are files in the incoming directory but the `archiver` option is disabled. This is something that could happen when the archiver is disabled in favor of the `streaming_archiver`, but PostgreSQL is still configured to ship WAL files to the incoming directory. This check is a non blocking check with the purpose of warning the user about the disk filling up with unarchived WALs. Closes: #80 2017-05-25 Giulio Calacoci Add `external_configuration` to `backup_options` It is now possible to instruct Barman that we are aware that configuration files reside outside of PGDATA, through the `external_configuration` value for the `backup_options` option. By doing so, users will suppress any warning regarding external configuration files during backup. 2017-05-23 Giulio Calacoci Command line options to enable/disable `get-wal` usage during recovery Add two options `--get-wal` and `--no-get-wal` to the `barman recover` command. They will control the usage of the `get-wal` option during the recovery, overriding what is specified in the configuration file. 2017-05-26 William Ivanski Documentation on parallel copy Updated documentation and man pages regarding parallel copy feature for both backup and recovery. 2017-05-22 Marco Nenciarini Archive .partial files coming from promotion of streaming source Change the way Barman handles multiple partial files during the archiving process. Archive older partial files as normal files, instead of aborting. Fix .gitignore syntax under sphinx directory 2017-05-12 Gabriele Bartolini Update TODO 2017-04-03 Mitchel Humpherys Document how to stop `cron` It's not immediately clear from the docs how to stop all of the processes that get started by the `cron` command. Add some instructions to the section on `cron` documenting how to stop things properly. Fixes #97 Documentation improvement: clarification about cron It can be a bit surprising to discover that barman doesn't have any sort of daemon program, as many long-running services do. Make this clear up front to avoid confusion. 2017-03-20 Cédric Villemain Documentation improvements 2017-05-05 Marco Nenciarini Ensure that PGCONTROL_CLASS items are copied as last step 2017-05-04 Marco Nenciarini Implement parallel copy for backup/recovery When `parallel_jobs` configuration option (or the `--jobs` run-time option) is set to a value greater than 1, the files to be copied are equally distributed between a number of buckets equal to the number of workers. The current maximum size of a bucket is 10GB. 2017-05-03 Marco Nenciarini In RsyncCopyController use an in-memory file_list instead of a file This provides enough information to the _generate_job to implement a job sharding strategy in a future commit. 2017-05-02 Marco Nenciarini Use a process pool to execute backup and recovery Add initial multiprocessing infrastructure to support parallel backup and recovery using rsync. Added the 'parallel_jobs' global/server configuration option, that specifies the number of parallel processes used for both backup and recovery operations. Default value is 1 (for retro-compatibility). Added also the `--jobs` and `-j` run-time option for the `backup` and `recover` commands to override the server behaviour. At the moment, when the value is set to 2 or more, copy operations are parallelized, but the logic to split them in smaller chunks is not yet implemented. 2017-04-18 Marco Nenciarini Recursively create directories during recovery Closes: SF#44 2017-04-05 Giulio Calacoci Reduce time ServerXLOGDBLock is held while archiving The ServerXLOGDBLock was held from the start to the end of the archiving process, but that was not how it was supposed to be. The implicit contract of a waiting lock such as the ServerXLOGDBLock requires that the lock is kept for the shortest possible time. However, that is not the case when archiving a large number of WAL files or when executing the pre/post archive hooks. This commit changes the acquisition strategy to acquire and release the lock on a per-WAL file basis. Fixes #99 2017-03-31 Marco Nenciarini Fix a docstring typo 2017-03-31 Giulio Calacoci Remove `tablespace_map` file during `recover` Barman takes care of tablespaces relocation during `recover` by creating the appropriate links into `pg_tblspc` directory. The eventual presence of `tablespace_map` file would instruct PostgreSQL to rollback these changes, so it is mandatory to remove this file during `recover`. For the same reason the code that was writing the `tablespace_map` file during `backup` was disabled. Fixes #95 2017-03-09 Marco Nenciarini Improve log messages when a backup fails 2017-02-27 Gabriele Bartolini Improve log message for streaming connections Add the server name in log messages when opening a streaming connection to PostgreSQL fails. Reported by Andreas Kretschmer 2017-02-06 Marco Nenciarini Properly detect broken Postgres connections When a Postgres cursor is required, issue a 'SELECT 1' query to check that connection is properly working. 2017-02-01 Gabriele Bartolini Cosmetic change in documentation 2017-02-01 Marco Nenciarini Reconnect to PostgreSQL if the connection drops Now Barman will try to reconnect automatically if the connection to PostgreSQL drops. Closes: SF#82 2017-01-20 Giulio Calacoci Add a init_check method to CheckStrategy Add a init_check method to the CheckStrategy object, the method will help keeping track of the check execution and produce a clear output in case of timeout failure. Set version 2.2a1 2017-01-05 Giulio Calacoci Update the ChangeLog file Set version to 2.1 2016-12-22 Marco Nenciarini Avoid importing barman.utils module twice 2017-01-03 Gabriele Bartolini Make retry hook script options global (bugfix) 2016-12-27 Marco Nenciarini Update the ChangeLog file 2016-12-26 Gabriele Bartolini Release 2.1a1 2016-12-21 Marco Nenciarini Add PostgreSQL 10 versioning scheme compatibility Fix #73 Exclude extraneous files during tablespace copy PostgreSQL stores the tablespace data in a directory defined in the PostgreSQL source as: "PG_" PG_MAJORVERSION "_" CATALOG_VERSION_NO The CATALOG_VERSION_NO is not easy to retrieve, so we copy "PG_" + PG_MAJORVERSION + "_*" It could select some spurious directory if a development or a beta version have been used, but it's good enough for a production system as it filters out other major versions. Fix #74 2016-12-20 Marco Nenciarini Align the list of excluded files to pg_basebackup Fixes #72 2016-12-18 Marco Nenciarini Return failure when 'barman get-wal' command is used on inactive servers The 'barman get-wal' command is meant to be used used as PostgreSQL's restore_command, so it must succeed only if the target WAL file has been actually returned. 2016-12-17 Marco Nenciarini Add last archived WAL info to diagnose output Show in diagnose the last archived WAL per timeline switch-xlog --archive waits for the closed WAL file Modify the switch-xlog --archive behaviour to wait for the specific WAL file, instead of a generic one. This is faster and it avoids possible issues when a higher timeline is present in the archive. 2016-12-03 Marco Nenciarini Add --archive and --archive-timeout options to switch-xlog command Improve the switch-xlog command adding the --archive and --archive-timeout options. The --archive option will start an archiver process for the selected server, and will wait for 30 seconds for a new xlog to be archived. The --archive-timeout TIMEOUT will allow the user to increase or decrease the number of seconds the archiver process will wait for the archival of a new xlog file, before exiting. 2016-11-26 Gabriele Bartolini Make streaming application name options global Make both 'streaming_archiver_name' and 'streaming_backup_name' global options, as per documentation. Bugfix. Fixes #57 2016-11-27 Marco Nenciarini Fix rsync failures due to files truncated during transfer Fixes #64 2016-11-26 Gabriele Bartolini Fix typo in documentation 2016-11-17 Gabriele Bartolini Convert README in README.rst Suggested by ckeeney. Closes #69 2016-11-24 Giulio Calacoci Correctly handle compressed history files Fix how barman behave when a compressed history files is found. Fixes: #66 Avoid dereferencing symlinks in pg_tblspc when preparing recovery This commit fixes a bug triggered by the presence of a broken tablespace symlink in the target pg_tblspc directory when recovery starts. Fixes: #55 2016-10-06 Martin Swiech Fix comparation of last archiving failure Compare the time of last failure instead of the WAL name to detect archiving failures. Closes: #40, #58 2016-11-13 Marco Nenciarini Avoid failing recovery if postgresql.conf is not writable Fixes: #68 2016-11-08 Giulio Calacoci Add start time and execution time to delete output Improve the output of `barman delete` command by adding some execution time statistics. Additionally, the utility function that prints human readable time deltas has been improved to correctly handle a wider range of values. Closes #67 2016-10-26 Ian Barwick Change references from non-existent `path` option to `path_prefix` 2016-10-28 Marco Nenciarini Exclude files within pg_replslot from backups Thanks to Magnus Hagander for reporting the issue. Ref: https://www.postgresql.org/docs/current/static/continuous-archiving.html#BACKUP-LOWLEVEL-BASE-BACKUP-DATA Fixes #65 2016-10-20 Giulio Calacoci Workaroud to setuptools testrunner bug. Ref links: * https://bitbucket.org/pypa/setuptools/issue/196 * https://github.com/pypa/setuptools/issues/196 2016-10-17 Giulio Calacoci Fix output of the replication-status command. The replication-status command used to emit an exception in some PostgreSQL releases, due to the different structure of the `pg_stat_replication` view. This patch fixes the issue checking for the existence of the fields before writing them to the output. Fixes #56 2016-10-11 Marco Nenciarini Move isort configuration in setup.cfg 2016-10-11 Gabriele Bartolini Add a note in docs about concurrent backup for 9.6 2016-09-30 Giulio Calacoci Split the logic of the copy method in CopyController This patch creates an analisys phase and a copy phase. the analisys phase is responsible for the preparation of the list of files that we want to copy. The copy phase is responsible for the execution of the actual copy using rsync. 2016-10-02 Gabriele Bartolini Fix level of section about WAL verification 2016-09-27 Giulio Calacoci Update sphinx index page 2016-09-26 Gabriele Bartolini Set version 2.1a1 2016-09-26 Marco Nenciarini Update the ChangeLog file 2016-09-26 Gabriele Bartolini Update architecture diagrams 2016-09-26 Marco Nenciarini Review of example configurations in doc/barman.d/ 2016-09-26 Leonardo Cecchi Fix image size in manual 2016-09-26 Gabriele Bartolini Set version to 2.0 2016-09-25 Gabriele Bartolini General review of the manual before 2.0 release 2016-09-23 Giulio Calacoci Add section `Upgrade from Barman 1.X` to manual. 2016-09-23 Leonardo Cecchi Improve documentation of get-wal command This patch improves the documentation of the get-wal command for the local recovery, specifying that the command must be executed as `barman` user and not as the user which is running the PostgreSQL server. It also includes details about the barman-wal-restore command. 2016-09-23 Gabriele Bartolini Update INSTALL, README and TODO files 2016-09-23 Leonardo Cecchi Add Windows server documentation Add a Windows server section in the "Setup a new server in Barman" chapter. 2016-09-21 Leonardo Cecchi Fix wrongly named max_replication_slots configuration parameter 2016-09-20 Marco Nenciarini Update the ChangeLog file 2016-09-19 csawyerYumaed Improve barman(5) manual page, adding conninfo Add additional information to documentation around conninfo. Closes #53 2016-09-20 Marco Nenciarini Set version to 2.0b1 Remove the quickstart guide At the moment, it is an empty document. It may be reintroduced later. 2016-09-20 Leonardo Cecchi Improve configurantion error message The configuration error message raised when a parameter has an invalid value was not telling the wrong value but only the key. This patch improves the error message in that case. 2016-09-16 Giulio Calacoci Add troubleshooting section to manual 2016-09-16 Leonardo Cecchi Improve "Features in detail" section 2016-09-19 Giulio Calacoci Avoid logging tracebacks in case of expected failures There was a few places in the code where an expected error was producing a traceback in the logs. They have been replaced with a simple error output. 2016-09-16 Giulio Calacoci Integrate barman-wal-restore in barman recovery During a remote restore, use the barman-wal-restore command instead of calling `barman get-wal` on the remote machine. It requires the barman-cli package to be installed on the target server. Force special tox environments to use a specific Python version Use Python 2.7 in the 'flake8' environment, because it doesn't work with Python 2.6. Use Python 2.6 in the 'minimal' envorinment, because it doesn't work with Python 3.x due to the presence of Python-2-only modules. Enable the minimal target in Travis-CI again, as it was failing due to the default Python version used there (Python 3.5). 2016-09-19 Marco Nenciarini Fix wrongly named database parameter in configuration examples Thanks to Abhijit Menon-Sen who found it. 2016-09-14 Leonardo Cecchi Add "server commands" documentation section This patch complete the section "Server commands" inside the Barman manual 2016-09-14 Giulio Calacoci Backup command section of the manual 2016-09-14 Leonardo Cecchi Add the "general commands" documentation section This patch complete the section "general commands" inside the Barman manual. 2016-09-13 Leonardo Cecchi Add the "setup new server" documentation section This patch complete the work on the "Setup of a new server in Barman" documentation section. 2016-09-13 Giulio Calacoci Make sure 'dbname' parameter is not set for streaming connections Also fix an issue in the parse_dsn method, many thanks to Théophile Helleboid for reporting it. 2016-09-15 Marco Nenciarini Fix flake8 plugin versions 2016-09-13 Marco Nenciarini Make sure path_prefix is honored in every Command subclass Commit ad8002745326fe20a2c4ce2a5e9115a26c5a5f49 contains a typo so it was not working at all. The typo is fixed in this commit. 2016-09-13 Giulio Calacoci Pass `--no-password` option to pg_basebackup and pg_receivexlog Barman will never send any password through the pg_basebackup
or pg_receivexlog standard input channel. Invoking them with the `--no-password` option will produce a cleaner error message in case of authentication failure. Modify tests accordingly. 2016-09-12 Giulio Calacoci Improve the warning message for configuration files outside PGDATA Now the `backup_method=postgres` code emits the warning message for configuration files that are located outside PGDATA only when that's the actual case. 2016-09-12 Gabriele Bartolini Moved barman-wal-restore in barman-cli project 2016-09-12 Marco Nenciarini Enable `show-server` command on inactive or disabled servers 2016-09-09 Marco Nenciarini Add missing release date on manual front page Disable minimal tox env on travis-ci as it doesn't work on py35 Update the ChangeLog file Prepare release 2.0a1 Run minimal and flake8 tox envs in travis-ci 2016-09-09 Gabriele Bartolini Fix flake8 error 2016-08-26 Gabriele Bartolini First stub of documentation for 2.0 This is a first attempt to change the documentation in Barman, which creates two artifacts: a manual and a quickstart guide. The work is currently incomplete, but it is ok for a first alpha release, given that all required information from a technical point of view is in man pages. The reason for this is that now Barman does not have a sequential installation procedure, but different options for backup and WAL archiving that produce several possibile combinations of setups. 2016-09-09 Marco Nenciarini Set last supported version by argh for Python 2.x Add minimal checks in Tox 2016-09-09 Gabriele Bartolini Release notes for 2.0a1 2016-09-09 Marco Nenciarini Rename barman-wal-restore.py to barman-wal-restore 2016-09-09 Leonardo Cecchi Fix exception in barman check Barman check raise an exception when streaming_archiver is on and streaming_conninfo is not configured. This patch controls if the streaming connection is correctly configured before checking the pg_basebackup compatibility, removing the exception cause. 2016-09-09 Marco Nenciarini Remove unit test dependency on environment Previously unit tests were failing the $USER environment variable was not defined. This patch removes this requirement. 2016-09-08 Leonardo Cecchi Make sure path is used in every Command class Switch xlog after a concurrent backup For a backup to be usable, all the XLOG files written between the backup start and the backup end must be archived. This patch makes PostgreSQL switch the current XLOG file after a concurrent backup either using rsync or streaming backup mode, in this way the WAL file containing the backup end position will be archived as soon as possible. 2016-09-07 Giulio Calacoci Add retry support when `backup_method=postgres` Modify the backup executor to handle the retry of a streaming backup with pg_basebackup (`backup_method=postgres`). Between every retry attempt, the PGDATA and the directories that contain the tablespaces are removed. Therefore, subsequent backup attempts restart from scratch. 2016-09-07 Gabriele Bartolini Fix E303 too many blank lines (flake8) 2016-09-07 Marco Nenciarini Support /etc/barman.d in rpm packaging 2016-09-01 Marco Nenciarini Redesign retry logic for backup and recovery The CommandWrapper class is now responsible to specifically design the logic of a retry operation in case of failure. Previously, retry was managed at an outer level, by the backup manager and the executor classes. With this new approach, every single command can now develop its own retry logic. 2016-09-05 Giulio Calacoci Force `archiver=on` when no archiver is active. This is a back-compatibility feature that will be kept in Barman for a few releases, and mitigate the upgrade to version 2.0 (which requires explicit setting of an archiving method - before, `archiver` was `on` by default) 2016-09-05 Gabriele Bartolini Change 'wal_level' to 'replica' in unit tests 2016-08-29 Ian Barwick Enable barman-wal-restore.py to pass -c/--config to Barman This enables barman-wal-restore.py to be used if the Barman server uses a non-default configuration file. 2016-09-02 Gabriele Bartolini Add slot name to `replication-status` (PostgreSQL 9.5+) Handle PostgresConnectionError exception differently Changed the way PostgresConnectionError works, in order to be more easily used in 'check' methods for both standard and streaming connection purposes. Specified also a new exception (PostgresAppNameError). 2016-09-01 Giulio Calacoci Remove any reference to Rsync.smart_copy() Remove the smart copy logic from the command wrappers module, as it is now part of RsyncCopyController. Support 9.6 syntax for `synchronous_standby_names` 2016-09-01 Gabriele Bartolini Cosmetic changes in recovery_executor.py Renamed also some methods as they are private 2016-08-30 Giulio Calacoci Introduce the CopyController module for recovery This patch introduces the use of the RsyncCopyController inside the recovery_executor.py module. 2016-08-31 Gabriele Bartolini Add Config.get_bwlimit() public method Move the private method _bwlimit() from RsyncBackupExecutor class into Config, and rename it into get_bwlimit(). 2016-08-30 Gabriele Bartolini Add status() method for archiver classes 2016-08-30 Marco Nenciarini Add StreamingWalArchiver._is_synchronous() Make code clearer and easier to read by encapsulating checks for synchronous replication in a separate method of the StreamingWalArchiver class. Make code more resilient by checking that PostgreSQL connection is working. 2016-08-25 Marco Nenciarini Introduce the CopyController module for backup This patch adds a new module called copy_controller.py. The new module is responsible for containing the logic of a physical copy of for a backup, starting with basic rsync support. This patch is a refactor of the existing code, with no additional functionality yet. Only backup executor is using this class for now. 2016-08-30 Marco Nenciarini Fix exception in 'check' when postgres is unreachable The check method of PostgresBackupExecutor and ConcurrentBackupStrategy objects were raising an exception if PostgreSQL connection was not working. This commit fixes this behaviour by correctly handling this case. 2016-08-26 Gabriele Bartolini Set default value of 'archiver' to 'off' This is a milestone in Barman's history as it removes the need for standard WAL archiving through the `archive_command` configuration option in PostgreSQL (even though it is still recommended to use it in conjunction with WAL streaming, for more robust architectures). **IMPORTANT**: When upgrading from 1.X.X versions of Barman to 2.X versions, users must explicitly add `archiver = on` (either globally or at server level). Created configuration templates for streaming and Ssh use cases 2016-08-24 Giulio Calacoci Enable streaming-only archiver with replication slots Remove the previous restriction that required standard WAL archiver (`archiver=on`) to be always present. Allow a server to operate with just WAL streaming if a replication slot is set (PostgreSQL 9.4 or higher is required). Replication slots make sure that every WAL file is streamed to the Barman server. Basic checks have been implemented at configuration level, requiring `streaming_archiver=on` and a `slot_name` if `archiver=off`. 2016-08-25 Leonardo Cecchi Improve replication slot error messages Barman didn't catch connection related exceptions in the `receive-wal` subcommand while creating and deleting a replication slot. This patch fixes that and reports to the user the connection error, avoiding to print a stack trace. This commit also adds a specific error message in case of drop of a replication slot that is currently in use. 2016-08-23 Giulio Calacoci Manage 'all replication slots are in use' error during slot creation Display an informative message to the user instead of a raw exception when the PostgreSQL's max_replication_slots setting is too low. 2016-08-23 Leonardo Cecchi Improve replication slot related messages This patch improves the replicaiton slot creation and drop messages quoting the names of the servers and of the replication slots. 2016-08-23 Giulio Calacoci Fix error messages during create and drop of replication slots Modified error message emitted in case of an already existing replication slot during the execution of receive-wal --create-slot. the message have been changed to error from info, replicating the behaviour of pg_receivexlog. Now, in case on error, the barman process exit code will be 1. The error message emitted during the execution of the receive-wal --drop-slot in case of a not existing replication slot have been changed to error as well. 2016-08-23 Marco Nenciarini Set version to 2.0 alpha 1 Anticipating PostgreSQL's new versioning system kicking in from version 10.0, development team has decided to adopt the same policy to Barman's versioning system starting from version 2.0. The development team has also decided that this version deserves to be 2.0 due to the scope of the new features that will be introduced, in particular streaming-only backup and "zero data loss" backups. 2016-08-23 Giulio Calacoci Improve error messages for receive-wal subcommand Error messages for the receive-wal subcommands were incoherent and this patch improves them, using a common form. In addition to that, the availability of the 'slot_name' configuration parameter was checked before the server version, making that check useless if the PostgreSQL server version has no support for replication slots. 2016-08-20 Gabriele Bartolini Support zero data loss backup (synchronous WAL streaming) Automatically start `pg_receivexlog` with `--synchronous` if the following conditions are met: - WAL streaming is active (see `streaming_archiver` and `streaming_conninfo`) - the WAL streamer application name (see `streaming_archiver_name`) is in the `synchronous_standby_names` priority list of the PostgreSQL server (which defines the synchronous standby servers) - PostgreSQL server 9.4 or higher - pg_receivexlog 9.5 or higher This feature allows a Barman backup solution to act like a synchronous standby server in a PostgreSQL business continuity cluster, bringing RPO=0 backups (zero data loss). Currently, RPO=0 requires a replication slot to be properly configured (see `slot_name`). 2016-08-23 Giulio Calacoci Fix exception during `barman diagnose` Barman diagnose was trying to serialize a LooseVersion object to the JSON stream, which is not serializable by default. This patch add the support for (Loose|Strict)Version objects to the BarmanEncoder object treting them as a string. 2016-08-22 Giulio Calacoci Fix archiver by catching ENOENT exception Fixed a corner-case error that could occour when an unexpected file (e.g. .partial file) was not found anymore by the archiver process while trying to move it in the errors directory. This commit also makes logging messages easier to read. 2016-08-17 Giulio Calacoci Add support for replication slots in `receive-wal` command If `slot_name` option is set and `streaming_archiver` is enabled, the `receive-wal` command will use the given replication slot to ensure that no WAL file is lost before its archival. The `check` command will monitor the status of the configured slot on the PostgreSQL side, accordingly with the value of the `streaming_archiver` option: if `streaming_archiver` is on, it will make sure that the slot exists and it is used, otherwise it will warn the user in case a stale slot is found. 2016-08-19 Giulio Calacoci Fix directory permissions when backup_method=postgres This patch fixes how pg_basebackup manages the permissions of the destination directory of a backup on the barman server, forcing 0700 at creation time. The same rights are applied to the destination directory of every tablespace copied. Fix misleading message on pg_receivexlog termination In case of termination of the pg_receivexlog subprocess using a signal, a misleading entry was written in the log file. This patch improves the error message. 2016-08-13 Gabriele Bartolini Introduce batch size for WAL archiving Previously, the `archive-wal` command would scan the incoming directories for both standard and streaming archiving and then process all the available WAL files in one single run. While this remains the default behaviour, two global/server options have been added to Barman: `archiver_batch_size` and `streaming_archiver_batch_size`. The first controls the batch size for the `archiver` process, the latter for the `streaming_archiver` process. The default value is 0, meaning unlimited batch (traditional behaviour in Barman). A value greater than 0 would trigger batch processing of WAL archiving in Barman, limiting a single run of `archive-wal` for the specific incoming queue (file archival or streaming) to that value. For example, setting `archiver_batch_size` to `10` would allow the `archive_wal` command to stop after processing a maximum of 10 WAL segments from `archive_command` in the same run (assuming `archiver` is enabled). In those contexts where `streaming_archiver` is enabled, setting `streaming_archiver_batch_size` to `10` would reach the same goal for WAL segments that have been streamed to Barman. These settings should be tuned based on the server's workload and the frequency of the `barman cron`. A more distributed WAL processing certainly reduces the probability of xlog.db locking (be careful not to set these value too low - the risk is that your Barman server stays behind the server's WAL production). Renamed WalArchiverBatch class into a more appropriate WalArchiverQueue. 2016-08-02 Leonardo Cecchi Add to receive-wal the '--drop-slot' option This patch adds to the 'receive-wal' Barman subcommand a '--drop-slot' option. That option drop the replication slot named in the 'slot_name' configuration parameter. The patch also adds the relative documentation to the Barman man page in the section 1. TODO: section in the tutorial that explains replication slots 2016-07-30 Leonardo Cecchi Add to receive-wal the '--create-slot' option This patch adds to the 'receive-wal' Barman subcommand a '--create-slot' option that creates a replication slot using the PostgreSQL streaming connection. The patch also adds the relative documentation to the Barman man page in the section 1. TODO: section in the tutorial that explains replication slots 2016-08-12 Leonardo Cecchi Correctly handle history files from streaming Since 9.3, `pg_receivexlog` follows timeline switches and creates appropriate `.history` files when needed. Previous implementation in Barman of WAL archiving from streaming connections erroneously skipped history files and treated them as errors. This patch fixes this behaviour. 2016-08-04 Leonardo Cecchi Avoid to lock XLOGDB while checking if WAL archiving is set up The patch avoid to take a lock during the checking of the WAL archiving setup. This is important for invocations of `barman check` to work correctly even when there are long activities on the XLOGDB file, i.e. when the archive process is managing many WAL files. 2016-07-29 Giulio Calacoci Fix flake8 version for Unit Tests 2016-07-28 Leonardo Cecchi Check tablespace-mapping option in pg_basebackup With `postgres` backup method, tablespaces are supported only for PostgreSQL servers 9.3 or higher and pg_basebackup 9.4 or higher. The reason is that pg_basebackup supports the `tablespace-mapping` option only from 9.4, and this option is needed in case there are tablespaces in the server. Given that pg_basebackup 9.4 is not compatible with PostgreSQL 9.1 and 9.2, these versions are not supported in case of tablespace presence. This patch adds a check for this scenario, providing users with warnings and preventing them from starting a backup in case the aforementioned requirements are not met. Documentation has been modified accordingly. 2016-07-19 Marco Nenciarini Add the `slot_name` configuration parameter Setting the `slot_name` parameter on a server running at least PostgreSQL 9.4 will enable the usage of a physical replication slot to track the position of the `receive-wal` process. 2016-07-19 Gabriele Bartolini Cosmetic changes in barman-wal-restore.py help page 2016-07-14 Marco Nenciarini Add a check for spurious messages in ssh_command output The aim of this check is to early detect potential communication problems through the configured ssh_command. For example, if the login script on the PostgreSQL server outputs any messages, this additional output could confuse Barman and therefore trigger random backup failures. 2016-07-13 Leonardo Cecchi Fix replication-status command on PostgreSQL 9.1 Barman was using the `pg_xlog_location_diff` function to calculate the difference in bytes between the master and the standby xlog locations. As `pg_xlog_location_diff` is available only from PostgreSQL 9.2, the `replication-status` command wasn't working on PostgreSQL 9.1. Adds the diff_lsn function to calculate the difference between two xlog locations. Fixes #45 2016-07-07 Marco Nenciarini 9.6 backup API support improvement PostgreSQL 9.6 provides functions to read the control file, so we can avoid parsing the backup_label to know the current timeline. 2016-06-16 Marco Nenciarini Support backup from standby This commit allows to backup standby servers using rsync or the new PostgreSQL 9.6 backup API. Allow invoking current_xlog_info on standby servers Previously, `pg_current_xlog_location` was used to get the current transaction log location, but this function can't be used on a standby server. Now the code checks the server type and use `pg_last_xlog_replay_location` if the server is in recovery mode. Uniform backup_label parsing The backup_label file was previouly parsed in multiple places of code. This code has been moved in a dedicated method. 2016-05-30 Gabriele Bartolini Add support for concurrent backups with PostgreSQL 9.6 As of version 9.6 PostgreSQL support concurrent backups natively. Add support to the new API, using new calls for concurrent backups. Users from PostgreSQL 9.2 to 9.5 can continue using the pgespresso extension. Code has been refactored accordingly. 2016-06-14 Giulio Calacoci Introduce PostgreSQLClient base class Refactor the PgBaseBackup and PgReceiveXlog classes, by moving common code in the PostgreSQLClient base class. Add PgBasebackup class unit tests. 2016-06-16 Giulio Calacoci Fix flake8 problem in tox See: https://github.com/gforcada/flake8-isort/issues/9 2016-06-13 Marco Nenciarini Refine the logic to detect unused WAL files. When multiple timelines are present in the archive, avoid purging WAL files that could be used by a backup on a different timeline. 2016-06-13 Giulio Calacoci Store detailed backup mode in backup.info file The `mode` field is present since long time, but before was always set to `default`. From now on, the `mode` field contains a descriptive string that shows the type of executor involved in the backup (rsync/postgres) and, if present, the strategy of the backup executor (exclusive/concurrent) 2016-06-03 Leonardo Cecchi Add a warning message to show-backup checking timelines If after a backup there is a timeline switch, the "Last WAL" field from the show-backup subcommand will show the last available WAL on the timeline on which the backup was taken and not the last archived one. This behaviour, while it's correct, can confuse the user. Now barman handles such situation and emits a warning if there is more than one timeline reachable from the displayed backup. 2016-06-10 Gabriele Bartolini Fix replication-status with no sent and write locations Fix bug with reuse_backup and backup_method=postgres 2016-06-09 Giulio Calacoci Add `streaming_backup_name` option When `backup_method` is set to `postgres`, the `streaming_backup_name` option sets the `application_name` of the managed `pg_basebackup` connection, improving monitoring of backups taken using `pg_basebackup`. By default, `streaming_backup_name` is set to `barman_streaming_backup`. Available for PostgreSQL 9.3 and above. 2016-06-05 Gabriele Bartolini Refine documentation and messages 2016-06-03 Giulio Calacoci Change psycopg2 cursor type to DictCursor Instruct the cursor object to use DictCursor objects to manage the results of the execute commands. Update calling methods accordingly. 2016-05-31 Giulio Calacoci Move exceptions in a dedicated module Exceptions have been moved in a dedicated `exceptions` module and organised into a hierarchy. 2016-03-30 Stefano Bianucci Base backups through PostgreSQL streaming replication Enable 'streamrep' only base backups with `backup_method=postgres` configuration option (global/server). Requires a valid streaming replication connection. Barman's implementation relies on `pg_basebackup`, therefore the following limitations apply: - `bandwidth_limit` is available with `pg_basebackup` >= 9.4 only - `network_compression`, `reuse_backup` and `tablespace_bandwidth_limit` are not supported - configuration files outside PGDATA are not copied 2016-05-30 Marco Nenciarini Add contrib/barman-wal-restore.py Python version of the WAL restore script to be used as 'restore_command' for recovery or fallback method with standby servers. This script invokes 'barman get-wal' through a secure shell channel, and supports also parallel fetching of WAL files (requires Barman 1.6.1+). Requires argparse. Supersedes barman-wal-restore for bash. 2016-05-27 Giulio Calacoci Limit the maximum execution time of 'barman check' command Add a configurable timeout for the execution of the 'barman check' command. It limits the maximum execution time to 30 seconds per server by default. The user can change this value using the 'check_timeout' configuration parameter. 2016-05-18 Giulio Calacoci Add version properties to PostgreSQL base class Add server_txt_version and server_major_version properties to the PostgreSQL class to quickly retrieve the major version of the PostgreSQL server. Refactor current_xlog_* properties in PostgreSQLConnection Introduce the current_xlog_info property that returns detailed info about current xlog location. Rename current_xlog property as current_xlog_file_name. Now current_xlog_file_name and current_xlog_location properties have been reimplemented and now they call current_xlog_info. Generalise DataTransferFailure exception A DataTransferFailure exception is not necessarily related to rsync only. Make it generic by accepting the command name as argument in the from_command_error factory method. Extract _purge_unused_wal_files method from SshBackupExecutor For reuse purposed, the code responsible for removing unused wal files in the backup method of SshBackupExecutor has been extracted. A new method called _purge_unused_wal_files is now part of the base class. 2016-05-11 Giulio Calacoci Use UnixRemoteCommand to execute remote operations When execute operation on remote server use UnixRemoteCommand instead of manually build an ssh Command. Some pre-existing UnixRemoteCommand invocations have also been fixed to honour the path_prefix setting. 2016-05-18 Marco Nenciarini Set version to 1.6.2a1 Update the ChangeLog file Prepared release 1.6.1 2016-05-16 Giulio Calacoci Clarify switch-xlog and replication-status failure output Change the output messages to be more clear regarding the cause of the failure. 2016-05-12 Gabriele Bartolini Fix traceback for switch-xlog with unknown server 2016-05-09 Marco Nenciarini Avoid logging tracebacks during WAL files deletion During WAL files deletion, if the target file doesn't exists, print a WARNING message and continue. 2016-05-06 Marco Nenciarini Update the ChangeLog file Prepared version 1.6.1a1 2016-05-06 Leonardo Cecchi Fix "PostgreSQL connection forced cleanup message" during unit tests 2016-05-06 Giulio Calacoci Modify manifest to include tutorial file in dist tar. 2016-05-06 Gabriele Bartolini Rolling out 1.6.1a1 Add "replication-status" command The "replication-status" command shows live information and status of any streaming client attached to one or more given servers. The --target option allows users to limit the scope to only hot standby servers ('hot-standby') or WAL streaming clients, such as pg_receivexlog ('wal-streamer'). By default, --target is set to 'all'. 2016-05-05 Gabriele Bartolini Check for 'backup_label' on the master If a PostgreSQL server is abruptly shut down when an exclusive backup is in progress, the 'backup_label' file will stay inside PGDATA. In case of a failed exclusive backup, if the Postgres server is down, "barman check" verifies that a backup_label is present. If so, prints "A 'backup_label' is present". Fixes #69 2016-05-04 Gabriele Bartolini Add check for empty WAL archive Add a check, visible in the check output only if fails, that war the user about a bad configuration of WAL shipping checking the dimension of the xlog.db file. 2016-05-03 Giulio Calacoci Add implementation of the switch-xlog command Implement the switch-xlog command that invokes pg_switch_xlog() on one or any PostgreSQL server. A --force option issues a CHECKPOINT just before the execution of the switch, forcing the server to generate a new WAL file. 2016-05-05 Marco Nenciarini Avoid logging a traceback during check with wrong ssh_command If ssh_command does not exists avoid logging an exception during checks with PostgreSQL versions older than 9.4 2016-05-02 Gabriele Bartolini Add `streaming_archiver_name` option When `streaming_archiver` is enabled, the `streaming_archiver_name` option properly sets the `application_name` of the managed `pg_receivexlog` process, improving monitoring of physical streaming replication clients. By default, `streaming_archiver_name` is set to `barman_receive_wal`. Available for PostgreSQL 9.3 and above. Remove redundant initial assignment 2016-04-28 Gabriele Bartolini Add reference to external project BarmanAPI 2016-04-22 Gabriele Bartolini Use absolute unit values in pretty_size() 2016-04-12 Giulio Calacoci Avoid crashing with a malformed conninfo parameter When a connection fails, prepend the error message with the connnection type ('conninfo' or 'streaming_conninfo'). 2016-04-09 Marco Nenciarini Fix usage of the --dbname option. The --dbname option is available only from version 9.3 of pg_receivexlog. This patch fixes the wal_archiver module, forcing barman when pg_receivexlog version is <= 9.2 to split the conninfo parameter. Fixes #36 2016-04-05 Marco Nenciarini Add timestamp and exit code to barman-wal-restore output 2016-04-01 Marco Nenciarini Make sure PostgreSQL connections are cleanly closed Modified the cli module to wrap every Server's top level command in a closing(server) context. Added also an atexit handler which logs a warning for stale connections. During backup command, the PostgreSQL connections are closed as soon as they are no longer needed. 2016-03-15 Leonardo Cecchi Make Barman unit tests remove temporary folders During the recovery executor unit tests, Barman was leaving a few temporary folders. This patch makes the unit tests remove them and covers a few corner cases where Barman, in consequence of a command failure, was not deleting the temporary folders. 2016-03-16 Rubens Souza Minor change in man page for streaming_conninfo 2016-03-14 Marco Nenciarini Fix warning notices during sphinx execution 2016-03-12 Christoph Moench-Tegeder Mark "failed backups" check as non-fatal Allow taking a new backup in case of presence of failed backups. This bug was introduced in version 1.6.0. 2016-03-11 Marco Nenciarini Rename '-x' get-wal option as '-z' The old '-x' remains as a deprecated undocumented alias. 2016-03-10 Leonardo Cecchi Add archive_mode=always support for PostgreSQL 9.5 This patch adds a basic archive_mode=always support for PostgreSQL 9.5. This archive mode is simply considered equal to "on". Thanks to Koichi Suzuki for opening a bug and proposing a fix. Fixes #32 2016-03-07 Marco Nenciarini Small comment fix in scripts/barman-wal-restore 2016-03-04 Gabriele Bartolini Set version to 1.6.1a1 2016-03-03 Gabriele Bartolini Check for PostgreSQL superuser privileges * Add the is_superuser property to PostgreSQLConnection * Add a related check for the server Closes #30 2016-02-26 Marco Nenciarini Add `--peek SIZE` option to `barman get-wal` When the `--peek SIZE` option is passed to `get-wal`, it will peek from the WAL archive the requested WAL file as well as subsequent files. 'SIZE' must be an integer >= 1. When invoked with this option, get-wal returns a list of zero to 'SIZE' WAL segment names, one per row. 2016-03-01 Marco Nenciarini Refine dependencies on external software Barman requires psycopg2 >= 2.4.2 to support streaming archiving 2016-02-26 Marco Nenciarini Update the ChangeLog file Prepared version 1.6.0 2016-02-24 Giulio Calacoci Fix subtle bug in local recovery 2016-02-20 Marco Nenciarini Depend on pytest-runner module only when tests are actually invoked 2016-02-18 Marco Nenciarini Enable E501 check (long lines) in flake8 Enabled in flake8 the E501 check, and reformatted text according to the new line length policy. Use pytest-runner module to run tests in setup.py 2016-02-18 Leonardo Cecchi Fix check if ssh is missing Catch any 'command not found' errors during ssh check 2016-02-17 Marco Nenciarini Update the ChangeLog file Prepared version 1.6.0b3 2016-02-15 Marco Nenciarini Stop running tests on Python 3.2 2016-02-11 Marco Nenciarini Make the code compatible with Python >= 3.2 All the incompatible constructs were rewritten to support both Python 2.x and 3.x. Previously we were running 2to3 during the setup execution. 2016-02-16 Gabriele Bartolini Add Leonardo Cecchi to authors 2016-02-15 Leonardo Cecchi Rollback transaction after starting a backup Following the complete refactoring of the PostgreSQL connection code, part of this 1.6.0 previous versions (alpha1, beta1 and beta2), Barman is keeping an IDLE connection while taking a backup. This could trigger spurious alerts if a transaction has been left open. 2016-02-15 Giulio Calacoci Fix output of 'barman check --nagios' Display the correct number of servers with 'barman check --nagios' 2016-02-12 Leonardo Cecchi Enhance test coverage for the wal_archiver module Remove the unused 'is_wal_relevant' method as well. 2016-02-12 Gabriele Bartolini Add section about contributing for Barman Add a section about .partial files in the documentation 2016-02-12 Leonardo Cecchi Add 'receive-wal running' check When `streaming_archiver` is enabled, this check ensures that a `receive-wal` process is running. 2016-02-12 Giulio Calacoci Add --reset option to receive-wal comand Add the `--reset` option to the `receive-wal` command, allowing users to reset the WAL location of the streaming archiver. This is useful in case Barman goes out of sync with the master server. 2016-02-10 Marco Nenciarini Update the ChangeLog file 2016-02-10 Gabriele Bartolini Prepared version 1.6.0b2 2016-02-05 Leonardo Cecchi Add check for FAILED backups The 'barman check' command now checks for the presence of failed backups. 2016-02-08 Giulio Calacoci Complete integration of RemoteStatusMixin class Let classes implementing the the get_remote_status method inherit from the RemoteStatusMixin base class, completing the refactoring activity started with a previous commit (3179a1e). 2016-02-08 Marco Nenciarini Fix unwanted removal of duplicate WAL from archive This commit removes an unused and dangerous option from the Compressor interface that allowed the removal of the source WAL file after compression (introduced in 1.6.0 development branch). Fixes: #27 Remove trashing of old WAL files from the archiver The archiver now always archives any file with a valid name that finds inside the incoming directory. NOTE: The block of code that has been removed changes behaviour of existing Barman installations, but in a conservative way. We defer to future versions of Barman to better understand WAL files' sequentiality (parsing history files, for example), by writing specific classes/modules responsible for deeper and more robust checks in the archive area. 2016-02-05 Marco Nenciarini Manage the termination of receive-wal in cron When the 'streaming_archive' option is disabled for a server, a cron execution makes sure that a running receive-wal process is correctly terminated. Clearer output and logging messages for `receive-wal`. 2016-02-05 Leonardo Cecchi Add check for archiver errors Barman now checks for the presence of files in the error directory. 2016-02-04 Giulio Calacoci Improve error handling when required server parameters are missing Add error handling in case a required parameter (i.e. ssh_command, conninfo, ...) is missing, by emitting meaningful error messages. 2016-02-02 Giulio Calacoci Adjust severity of some messages produced by cron command The messages about subprocess execution produced by the cron command were emitted for every cron run. They provide little information and their presence may make the log difficult to analyse, so they have been removed from log. 2016-02-02 Aydan Update docs - Added remote recovery command Closes: #26 2016-02-01 Marco Nenciarini Update the ChangeLog file Prepared version 1.6.0b1 2016-01-29 Marco Nenciarini Fix python 3.2 error on travis-ci Avoid exceptions during ProcessManager initialization Previously a LockFileParsingError exception could have been raised if a lock file were empty. Also, the LockFile class now writes the lock content atomically 2016-01-28 Giulio Calacoci Remove a spurious error when last_failed_wal is a history file Fixes: SF#77 2016-01-27 Giulio Calacoci Fix the 'status' command output with PostgreSQL < 9.4 Make sure only one 'Last archived WAL' line is present. Fix the command used to retrieve the last archived WAL through ssh when the pg_stat_archiver view is not available. Fixes: #25 2016-01-28 Gabriele Bartolini Set version to 1.6.0b1 2016-01-21 Giulio Calacoci Add --stop option to receive-wal command Add the option --stop to the receive-wal command, allowing the user to stop a receive-wal process that running in background for a specific server. The implementation is based on the new class ProcessManager that will be used in the future to manage all barman's sub-processes. 2016-01-24 Gabriele Bartolini Add current data size of the server in 'barman status' 2016-01-22 Gabriele Bartolini Updated NEWS with current code 2016-01-19 Giulio Calacoci Improve archive-wal robustness and log messages Display clearer error messages when discarding a WAL file. Also, WAL files that are older than the first backup are now moved in the 'errors' directory: FAILED backups are now ignored in this process. TODO: Add a method to check the presence of files in the error directory. Fixes: #24 2016-01-21 Marco Nenciarini Move archive-wal code into the wal_archiver module 2016-01-17 Gabriele Bartolini Reorder options in barman.5 man page 2016-01-15 Marco Nenciarini Add include sorting check in tox flake8 environment 2015-12-05 Giulio Calacoci Implement pygzip and pybzip2 compressors The new pygzip and pybzip2 compressors are implemented using Python's internal modules for compressing and decompressing gzip and bzip2 files. This avoids the cost of forking and executing the shell. Initial idea by Christoph Moench-Tegeder 2016-01-14 Marco Nenciarini Update copyright years to 2016 2016-01-08 Giulio Calacoci Add 'receive-wal' cron integration Add integration of the new command 'receive-wal' during the execution of the cron command. If the option 'streaming_archiver' is on, the command 'receive-wal' is automatically executed as subprocess by the cron command, allowing Barman to receive WAL files from PostgreSQL using the native streaming protocol. TODO: Implement a way to interrupt a running streaming process 2015-12-09 Stefano Zacchiroli Add support for the pigz compressor Pigz on-disk format is the same of gzip, but (de)compression is much faster due to the usage of CPU parallelism. For write-intense workloads pigz vs gzip might make the difference, between struggling to keep up with archiving incoming WALs and being able to keep up easily. 2016-01-09 Gabriele Bartolini Add check() method in WalArchiver classes Move code from the server.check_postgres() method in the WalArchiver class, so that each implementation (FileWalArchiver and StreamingArchiver) perform their specific checks. Created the RemoteStatusMixin class that implements a cacheable version of remote status information. Currently it is only used by the WalArchiver class tree. TODO: propagate it to all other classes implementing the get_remote_status() function in a later commit. 2016-01-08 Christoph Moench-Tegeder Permit `archive_mode=always` for 9.5 servers PostgreSQL 9.5 introduces archive_mode=always. Even though Barman's current framework for checks does not allow yet to correctly monitor archiving from standby servers, Barman now allows users to set `archive_mode` to `always`, provided that archiving happens from the master. 2016-01-08 Giulio Calacoci Fix server creation and output of check command Modified the init method of Server Object, managing errors/exception using the msg_list attribute of the configuration. Modified the check command output. 2016-01-04 Giulio Calacoci Rename get_first_backup and get_last_backup methods Renamed get_first_backup() and get_last_backup() methods of the Server and BackupManager classes, respectively into get_first_backup_id() and get_last_backup_id(), for coherence and code clarity. Now the fact that the method returns the ID of a backup instead of a BackupInfo object has been made explicit. Fix management of backup id during backup deletion Fix an error that prevents the deletion of an existing backup while a backup on the same server is in progress. Fixes #22 2015-12-31 Marco Nenciarini Use isort python package to keep imports ordered 2015-12-14 Giulio Calacoci Implement `receive-wal` command The `receive-wal` command receives the stream of transaction logs for a server. The process relies on `pg_receivexlog` to receive WAL files from a PostgreSQL server through the streaming protocol. It is a foreground process which lasts until it receives a SIGINT. 2015-12-31 Marco Nenciarini Add support for long running Commands The Command class now supports the execution of long running subprocesses, by passing the subprocess output and error streams to a handler, line by line. Remove spurious newlines from log lines Error messages coming from libpq often terminate with a newline. Always call strip() on strings that have been extracted from exceptions before logging them. Make get_remote_status methods behaviour coherent A 'get_remote_status' method should never raise an exception in case of error. The expected behaviour is to set missing values to None in the resulting dictionary. 2015-12-29 Leonardo Cecchi Fix exception when streaming connection is unavailable Previously, when 'streaming_archiver' was on and pg_receivexlog correctly installed, Barman would raise an AttributeError exception when streaming connection to PostgreSQL was unavailable, as it could not detect the server version. This commit fixes the error by modifying the management of a failed connection to PostgreSQL through streaming connection. It also adds an unit test for this scenario, as well as hints for the user during `barman check` suggesting a possible streaming connection error. 2015-12-28 Leonardo Cecchi Propagate "-c" option to any Barman subprocess The "-c" option lets users specify a custom location for the configuration file. It must be propagated to every Barman subprocess for consistency of behaviour. Fixes #19 2015-12-24 Gabriele Bartolini Update documentation about WAL archiving 2015-12-22 Marco Nenciarini Manage WAL duplicates in `archive-wal` command In case a WAL file is archived twice, `archive-wal` will check if the new file is identical to the already archived one (compressed WAL files will be decompressed and checked). If yes, the second file is discarded, otherwise it is saved in the 'errors' directory. Accept compressed WAL files in incoming directory In case a WAL file in the incoming directory is already compressed using a known algorithm, archive it as is. Add streaming support to 'archive-wal' command The `archive-wal` command now looks for both archived WAL files (shipped from PostgreSQL through its `archive_command`) and streamed ones (via the `receive-wal` command). 2015-12-24 Giulio Calacoci Add sleep time option to `barman-wal-restore` script Introduce the '-s seconds' option to the barman-wal-restore script. After a failure of the get-wal command, the script will sleep for the specified amount of seconds (default 0, no wait) before exiting the restore_command from PostgreSQL. Closes #7 2015-12-22 Marco Nenciarini Add `streaming_wals_directory` and `errors_directory` options Make sure WAL archiving is atomic. Fixes: #9,#12 2015-12-15 Marco Nenciarini Get rid of pytest warnings from pytest-catchlog 2015-12-11 Marco Nenciarini Restore compatibility with PostgreSQL 8.3 On PostgreSQL older than 8.4, disable the query that detects those settings coming from an included file - as the required pg_setting field was only included in that release. 2015-11-24 Marco Nenciarini Fixes possible unhandled exceptions in barman.postgres Improve the testing coverage of barman.postgres module. 2015-12-06 Christoph Moench-Tegeder Remove redundant sete[gu]id() calls Python's documentation of the os.setuid/os.setgid calls is oversimplified to the point of being wrong. Under the hood, the setuid(2)/setgid(2) functions are used, thus making these calls as POSIX compliant as the underlying operating system. As barman won't be installed SUID (it's an interpreter file), effective, real and saved user id should always be the same. That way, the only reason to use setuid() family calls at all is for a privileged user (root) to change over to the barman (unprivileged) user. According to POSIX.1-2013, the setuid() call changes all three (real, effective and saved) uids if called by a privileged user (the same holds for setgid() and the group ids). In this construct, the additional seti[gu]id calls are redundant no-ops and can be removed. 2015-11-27 Giulio Calacoci Add support for 'streaming_archiver' option Add a global/server boolean option called 'streaming_archiver' which enables the use of the PostgreSQL's streaming protocol to receive transaction logs from a server. If set to 'on', Barman expects to find `pg_receivexlog` in the PATH (see `path` option) and that streaming connection for the server is properly working. Subsequently, it activates both proper connection checks and management of WAL files. Refactored the code to use a new class, the StreamingWalArchiver, which is responsible for managing WAL files using a streaming connection. Add support for 'archiver' option Add a global/server boolean option called 'archiver' which enables continuous WAL archiving (through log shipping using PostgreSQL's archive_command) for a specific server. If set the 'true' (default), Barman expects that PostgreSQL continously deposits WAL files in the 'incoming' directory of that server, and subsequently activates proper checks and management. Refactored the code to use a new class, the FileWalArchiver. The class is in charge of both archival and compression of WAL files. This is the first step towards the implementation of streaming archiving support. 2015-11-25 Marco Nenciarini Reorder constants in barman.config module 2015-11-20 Giulio Calacoci Add `streaming_conninfo` server option Added the `streaming_conninfo` option to Barman configuration. This option (with server scope only), adds the support in Barman for a streaming replication connection to a PostgreSQL server. Added also a simple check for the barman check command, that tests that the streaming connection to PostgreSQL is properly working. 2015-11-13 Giulio Calacoci Add support for `path_prefix` configuration option The `path_prefix` global/server option can contain one or more absolute paths, separated by colon, where Barman looks for executable files. The paths specified in `path_prefix` are tried before the ones specified in `PATH` environment variable. 2015-11-10 Gabriele Bartolini Refactored access to PostgreSQL Created the 'postgres' module which is responsible for any communication with a PostgreSQL server, wrapping all connection and query management activities. This module is the only point of access to PostgreSQL. 2015-11-06 Stefano Bianucci Rename `copy_method` configuration key to `backup_method` 2015-11-03 Giuseppe Broccolo Create an implicit restore point at the end of a backup At the end of a backup, if the PostgreSQL version is 9.1 or newer, create an implicit restore point using the pg_create_restore_point() function. 2015-11-11 Gabriele Bartolini Set version to 1.6.0a1 2015-11-13 Marco Nenciarini Update the ChangeLog file Prepared version 1.5.1 2015-11-13 Giulio Calacoci Create pg_xlog/archive_status directory at the end of recovery On PostgreSQL 8.3, the pg_xlog/archive_status directory is not automatically created, if missing, during the startup. To avoid errors Barman now creates that directory at the end of any recovery operation. 2015-11-13 Marco Nenciarini Converted sphinx directory README to Markdown 2015-11-10 Marco Nenciarini Pin (temporarily) pytest-catchlog to version 1.1 The pytest-catchlog release 1.2.0 broke Python 2.6 compatibility, so pin the 1.1 version until it will be restored. Ref: https://github.com/eisensheng/pytest-catchlog/pull/15 In latest release of pytest-catchlog the result of caplog.records() call is a copy of the event list so any modification of that has no global effect. Change the code that uses caplog.records() to not rely on the previous undocumented behaviour. 2015-11-05 Marco Nenciarini Fix a typo in barman-wal-restore help output 2015-11-04 Marco Nenciarini Update the ChangeLog file Avoid forcing arcfour cypher in barman-wal-restore OpenSSH 6.7 dropped many low security algorithms, and arcfour is one of them. So we stop forcing it in the script, allowing the user to define the wanted algorithm in ~/.ssh/config Ref: http://www.openssh.com/txt/release-6.7 2015-11-04 Giulio Calacoci Improve error messaging for missing config files 2015-11-04 Gabriele Bartolini Set version to 1.5.1b1 2015-10-27 Giulio Calacoci Manage 'wal_level' only for PostgreSQL >= 9.0 Really fixes #3 2015-10-26 Marco Nenciarini Fix all E731 pep8 errors Always use a def statement instead of an assignment statement that binds a lambda expression directly to a name. See: https://www.python.org/dev/peps/pep-0008/#programming-recommendations 2015-10-22 Marco Nenciarini Update the ChangeLog file 2015-10-22 Gabriele Bartolini NEWS for 1.5.1a1 Set version to 1.5.1a1 Revert version to 1.5 branch due to relevant bug fixing 2015-10-12 Giulio Calacoci Add the 'archive-wal' command The 'archive-wal' command executes the WAL maintenance operations on a given server. This command is automatically executed as a subprocess by the cron command, allowing the parallel execution of WAL archiving on different servers. 2015-10-20 Giuseppe Broccolo Avoid 'wal_level' check on PostgreSQL version < 9.0 Conditionally skip 'wal_level' check in PostgreSQL versions prior to 9.0. Fixes #3 Avoid retention policy checks during the recovery Additionally, skip any missing tablespace directory during the deletion of a backup (like we already do with missing pgdata directory) 2015-10-20 Marco Nenciarini Fix more incorrect mock assert calls in unit tests Add flake8-mock plugin in flake8 tox environment to make sure it will not happen again. Some more cleanup of testing infrastructure has been done: * Switch from pytest-capturelog (unresponsive upstream) to pytest-catchlog. * Remove the version pinning from mock as 1.3.0 supports py2.6 again. * Add flake8-copyright and flake8-string-format plugins. 2015-10-09 Giulio Calacoci Allow parallel cron execution for different servers Allow the execution of multiple 'barman cron' processes, each one handling a different server. Servers will be handled sequentially, skipping those that are already being served by another cron process. 2015-10-22 Marco Nenciarini Fix calculation of backup size In Barman 1.5.0 the internal directory structure of a backup has been changed, moving tablespaces from being inside the data directory to being inside the backup directory. The method backup_fsync_and_set_sizes() now takes that into account by running against the whole backup directory. Fixes #5 2015-10-12 Marco Nenciarini Fix some incorrect mock assert calls in unit tests 2015-10-06 Giulio Calacoci Support for mixed compression types in WAL files (#61) Decompress each WAL file using the specific algorithm reported in the XLOG archive (xlogdb). Improve management of xlog.db errors Better management of errors during the decoding of the name of an xlog segment. Added a hint that suggests to run "barman rebuild-xlogdb" to fix corruptions in the xlogdb file. 2015-10-02 Gabriele Bartolini Started version 1.6.0a1 2015-09-25 Marco Nenciarini Update the ChangeLog file Update Copyright notice 2015-09-25 Gabriele Bartolini Remove obsolete section from tutorial Removed a section in the tutorial regarding the 'delete' command, as pointed out by Ian Barwick. 2015-09-25 Marco Nenciarini Enable flake8 in every tox run 2015-09-24 Gabriele Bartolini Prepared version 1.5.0 for final release 2015-09-24 Marco Nenciarini Update tox.ini to allow specifying a target test Also add .cache directory, which is created by latest tox, to .gitignore 2015-09-24 Giulio Calacoci Changed import of call() method. Now is compatible with all versions of py.test libraries 2015-09-16 Giulio Calacoci Support for non-critical checks during backup Add a filter for non-critical checks during backup operations. This patch fixes a bug that prevented users from taking a new backup when minimum redundancy or smelly backup checks failed. 2015-09-15 Marco Nenciarini Update the ChangeLog file 2015-09-15 Gabriele Bartolini Prepared for 1.5.0 alpha 1 2015-09-09 Gabriele Bartolini Converted man pages to Markdown 2015-09-08 Gianni Ciolli Some small fixes to the tutorial 2015-09-01 Francesco Canovai Updated spec file for RPM building 2015-09-04 Giuseppe Broccolo Add barman-wal-restore script Add a simple bash script to be used as `restore_command` in a standby scenario, as fallback method. The script uses ssh to connect to the Barman server and requests the required WAL file for recovery. The script checks that destination path is not a directory. 2015-08-27 Gabriele Bartolini Convert tutorial to Markdown 2015-08-28 Giulio Calacoci Manage options without '=' in PostgreSQL configuration files 2015-08-18 Gabriele Bartolini Allow 'pre' retry hook scripts to stop the command Add EXIT_ABORT_STOP (63) and EXIT_ABORT_CONTINUE (62) exit codes to control how a retry hook script is aborted. Termination of a retry hook script with EXIT_ABORT_CONTINUE informs Barman to continue with its operations. By terminating a retry hook script with EXIT_ABORT_STOP, users request Barman to stop its main operation (i.e. backup or WAL archival). EXIT_ABORT_CONTINUE is implemented by every retry hook script. EXIT_ABORT_STOP is currently implemented only with 'pre_backup_retry_script' and 'pre_archive_retry_script'. EXIT_ABORT_STOP is currently ignored by 'post_backup_retry_script' and 'post_archive_retry_script', and its behaviour in these cases is identical to EXIT_ABORT_CONTINUE. 2015-08-21 Marco Nenciarini Add flake8 tox environment 2015-08-18 Gabriele Bartolini Documentation for 'barman_lock_directory' 2015-08-18 Giulio Calacoci Analyse include directives for PostgreSQL Check if PostgreSQL administrators take advantage of include directives and specify additional files. Include directives (include, include_if_exists, include_dir) are not supported in Barman for files that reside outside PGDATA. During backup, warn users and list the files that require manual backup. During recovery, warn users about the presence of include directives in PostgreSQL configuration. 2015-08-18 Gabriele Bartolini Workaround for rsync on SUSE Linux Many thanks to Christoph Moench-Tegeder for his workaround proposal. This fixes the outstanding issue on SUSE Linux - previously raised by issues #13 and #26. See also: https://bugzilla.opensuse.org/show_bug.cgi?id=898513 2015-08-17 Gabriele Bartolini Added copy_method option (fixed to rsync) 2015-08-06 Giulio Calacoci Support for 'retry' hook scripts for backup and WAL archiving A 'retry' hook script is a special kind of hook scripts that Barman tries to run indefinitely until it either returns a SUCCESS (0) or an ABORT (63) exit code. Safe hook scripts are executed immediately before (pre) and after (post) the command execution. Standard hook scripts are executed immediately before (pre) and after (post) the retry hook scripts. This patch adds support for pre/post retry hook scripts for backup and WAL archiving. The following four global/server configuration options have been added: * pre_backup_retry_script: executed before a backup * post_backup_retry_script: executed after a backup * pre_archive_retry_script: executed before a WAL archive operation * post_archive_retry_script: executed after a WAL archive operation By default, no retry hook script is executed. Environment variables are identical to the equivalent standard hook script. 2015-08-12 Gabriele Bartolini Updated AUTHORS file 2014-05-29 Giulio Calacoci Support for the 'get-wal' command The 'barman get-wal' command allows users to fetch any WAL file from the archive of a specific server in Barman. 2015-08-21 Marco Nenciarini Add simple Travis CI integration 2015-08-11 Gabriele Bartolini Set version 1.5.0a1 2015-07-30 Giulio Calacoci Preserve Timeline history files. Fixes #70. Added check for the management of history files during the removal of a backup or during normal maintenance operations (cron). 2015-08-06 Giulio Calacoci Forbid the delete of a running backup Block the execution of a barman delete command on a backup that is in progress. Use locks to ensure that there are no running backups on that server. Otherwise check for the position of the backup to be deleted in the catalogue and its status: if it is the first backup and its status is STARTED or EMPTY, the backup is running and delete is forbidden. 2015-08-03 Giulio Calacoci Follow symlinks when checking directory paths Check conflicting paths in configuration files using canonical paths and following symlinks when necessary. 2015-07-13 Francesco Canovai Pin mock==1.0.1 for testing. In version 1.1.x mock has changed some behaviours and is currently incompatible with our unit tests. 2015-07-08 Stefano Bianucci Execute check() before starting a backup Execute the full suite of tests from barman check command before starting a backup. Skip the execution of a backup in case check fails. Add a strategy for managing the results of the checks. Now every check is properly logged as error (failure) or debug (success). 2015-07-03 Giulio Calacoci Support code documentation using Sphinx Generate code documentation using Sphinx autodocs 2015-07-13 Giulio Calacoci Modified test for correct management of timezone 2015-07-10 Marco Nenciarini Properly support BackupInfo serialization in JSON format. 2015-07-01 Giulio Calacoci Improved management of configuration in tests Improved and simplified the management of configurations in tests. Added a method for the creation of dictionaries containing all the keys that are usually present in a configuration. Updated tests accordingly. 2015-06-18 Giulio Calacoci Second part of the backup_executor module's refactor Refactored and streamlined the executor module. Introduced a specific class for physical backup with Rsync through Ssh, as well as a strategy pattern for the management of exclusive and concurrent backups (for backups from a standby). 2015-06-23 Gabriele Bartolini Standard error management for server commands Standardised the process of managing errors with server commands in barman/cli.py. By default, inactive servers are skipped (without error) as well as temporarily disabled servers (with error). No distinction is made between calling a command with one server as target or with a list of them (including 'all'). Exceptions are check (no server is skipped, errors are returned only for active servers), cron (no error is ever returned), list-server and diagnose (both managing active/disabled servers with no errors). Inactive servers are the ones with 'active' option set to False. Disabled servers are the ones with internal directory conflicts (e.g. WALs directory = base backup directory). 2015-06-23 Gabriele Bartolini Asciidoc support for man pages and tutorial 2015-06-16 Giulio Calacoci Fixed error in WAL rate calculation. Solved an error during the evaluation of the WAL rate for a backup. Added two basic unit tests. 2015-06-12 Stefano Bianucci Add check for wal_level For better usability, warn users about setting a proper value for wal_level setting in PostgreSQL. 2015-05-14 Stefano Bianucci Add check for conflicting paths in Barman's configuration Added controls for path-clash during the creation of Barman servers. If there are conflicting paths, Barman will disable those servers containing errors. If a potentially destructive command like "backup" is issued on a server containing conflicts, Barman exits with an error message. 2015-05-21 Giulio Calacoci Complete refactor of the 'recover' command The main 'recover' command has been completely refactored, through the creation of a separate module, called 'recovery_executor'. The RecoveryExecutor class now embodies both local and remote operations, laying the road for future improvements. This patch also fixes #68, by disabling dangerous settings in postgresql.auto.conf (available from PostgreSQL 9.4). Basic unit tests have been added. 2015-05-19 Giulio Calacoci Rename class Server to ServerConfig in barman.config module Previously both barman.config and barman.server modules had a Server class. The former has now been renamed to ServerConfig, hence removing the ambiguity. 2015-05-21 Marco Nenciarini Fix incompatibility with tox version >= 2 2015-05-08 Giulio Calacoci Make sure that even an EMPTY backup has a server set. 2015-05-07 Giulio Calacoci Improve xlog.db integrity (Closes: #67) * Execute flush() and fsync() after writing a line in xlog.db * Execute fsync() on incoming directory after every WAL is archived 2015-04-13 Giulio Calacoci Remove unused 'server' argument from WalFile.from_xlogdb_line() This also fix regression in 'barman delete' command introduced with commit 7ac8fe9c41fd7e5636f370abdc92ca785057263e. 2015-04-13 Giulio Calacoci Fix exception during error handling in barman recovery (Closes: #65) 2015-03-24 Giulio Calacoci Improved cache management of backups Streamlined the management of the cache of the backups. It is now possible to register and remove a backup from the cache. The cache structure has been simplified and now it is a simple dictionary of backups. Status filters are applied by the get_available_backups() method. Managed registration and removal of a backup in the cache during backup operations. 2015-03-02 Giulio Calacoci Create backup_executor module and refactor backup. Extract all the methods relative to the execution of a backup into a dedicated object. The RsyncBackupExecutor encapsulates the operation of backing up a remote server using rsync and ssh to copy files. 2015-03-27 Marco Nenciarini Improved management of xlogb file The xlogdb file is now correctly fsynced when updated. Also, the rebuild-xlogdb command now operates on a temporary new file, which overwrites the main one when finished. 2015-03-27 Stefano Bianucci Add "active" configuration option for a server It is now possible to temporarily disable a server through the 'active' configuration option. Defined at server level as a boolean, when set to False the server is ignored by main operational commands such as cron, backup and recover. By default, it is set to True. 2015-03-20 Giulio Calacoci Modified Barman version following PEP 440 rules. Changed Barman version from 1.4.1-alpha1 to 1.4.1a1 following PEP 440 rules. Adapted RPM build scripts to produce packages with the correct names. 2015-03-17 Giulio Calacoci Fix computation of WAL production ratio The WAL file ratio reported in the 'show-backup' command output was wrong because it was considering only the number of WALS produced during the backup instead of the number of WALs produced until next backup. 2015-02-25 Giulio Calacoci Fix for WAL archival stop working if first backup is EMPTY In some rare cases, if an empty backup has left by a failure during a backup, the cron could start trashing WAL files, as if there is no available backups. Closes: #64 2015-03-06 Gabriele Bartolini Added 'barman_lock_directory' global option Barman now allows to specify the default directory for locks through the global option called 'barman_lock_directory', by default set to 'barman_home'. Lock files will be created inside this directory. Names of lock files have been slightly changed. However, this won't affect users of Barman, unless you are relying on their locations and paths. This patch introduces four classes for managing lock files: GlobalCronLock, ServerBackupLock, ServerCronLock and ServerXLOGDBLock. 2015-02-03 Giulio Calacoci New backup directory structure Inside the backup directory the 'pgdata' has been renamed to 'data'. Tablespaces, if present, are stored into individual directories alongside the 'data' directory. During backup and recovery operations, tablespaces are copied individually using a separate rsync command. 2015-02-12 Giulio Calacoci Improve backup delete command Improve robustness and error reporting of backup delete command 2015-02-10 Stefano Bianucci Add unit tests for dateutil module compatibility 2015-02-08 Gabriele Bartolini After a backup, limit cron activity to WAL archiving only (#62) 2015-01-28 Marco Nenciarini Clarify rpm spec comments about pre-releases 2015-01-28 Gabriele Bartolini Updated backlog (TODO list) 2015-01-23 Marco Nenciarini Update metadata in setup.py - Improve barman description - Add Python 3.4 2015-01-23 Gabriele Bartolini Started version 1.4.1-alpha.1 Update the ChangeLog file Prepared version 1.4.0 2015-01-20 Francesco Canovai Updated spec files for RHEL7 2015-01-16 Giulio Calacoci Delete basebackup dir as last action of a delete. Split the delete operation: remove the PGDATA directory first, then the related WAL files and, at last, the basebackup directory. 2015-01-13 Giulio Calacoci Add minimum_redundancy tests in test_retention_policy.py 2015-01-13 Gabriele Bartolini Fix calculation of deduplication ratio 2015-01-12 Gabriele Bartolini Update the ChangeLog file Prepared documentation for version 1.4.0-alpha1 2015-01-11 Gabriele Bartolini Store deduplication effects for incremental backup When incremental backup is enabled and uses hard links (reuse_backup = link), output of 'backup' command reports the effects of deduplication. The metrict is stored along the backup.info file in the 'deduplicated_size' field. IMPORTANT: this metric refers to the increment in size of the current backup from the previous backup and reflects only the situation at backup time. 2015-01-10 Gabriele Bartolini Prepared version 1.4.0-alpha1 Updated copyright to 2015 2015-01-09 Marco Nenciarini Fix smart_copy of tablespaces when using bwlimit option 2015-01-07 Giulio Calacoci Add dedicated exception for PostgreSQL connection errors 2015-01-08 Giulio Calacoci Fix missing argument error in retention policies backup_status method Improve test coverage for retention_policy.py module 2015-01-07 Giulio Calacoci Remove logging of tracebacks on error during backup 2015-01-05 Gabriele Bartolini Avoid logging of tracebacks during smart copy While retrieving the list of files on destination for smart copy, log any failure as error instead of exception 2014-12-22 Giulio Calacoci Unit tests for BackupInfo object 2014-12-24 Giulio Calacoci Change the way BackupInfo are created for testing Merge the method build_test_backup_info and the mock_backup_info. Now we use real BackupInfo objects instead of a Mock 2011-12-07 Marco Nenciarini Incremental base backup implementation Add support for reuse_backup global/server option, accepting three possible values: * off: no incremental backup support (default) * copy: uses the last available backup as source (preventing unmodified files from being copied) * link: same as copy but uses hard links on destination, if the filesystem on the backup server allows it (reducing the occupied space) Add support for command line '--reuse-backup' option (default: link). 2014-12-24 Gabriele Bartolini Allow last_archived_wal to be any xlog file Correctly show any xlog file as last_archived_wal for pre-pg_stat_archiver cases. Improve testing and docstrings for barman/xlog.py module. 2014-12-09 Giulio Calacoci Improve robustness of ssh_command and conninfo options 2014-12-18 Giulio Calacoci pg_stat_archiver support for PostgreSQL 9.4 Integrate pg_stat_archiver with PostgreSQL 9.4 servers for the barman check, show-server and status commands. 2014-11-28 Giulio Calacoci Improve robustness of retention policy unit tests 2014-12-16 Giulio Calacoci Fixes retention policies delete bug (#58) The method responsible for deleting obsolete backup in retention policies enforement, will not raise anymore the 'NoneType object is not iterable'. This prevents barman from terminating abruptly. 2014-11-28 Giulio Calacoci Pass list of available backups to retention policy code 2014-12-02 Giulio Calacoci Include history files in WAL management 2014-12-04 Giulio Calacoci Added a util method to find an executable in system path. If rsync is not present on system, a proper error message is displayed to the user when a command using rsync is issued 2014-12-09 Giulio Calacoci Changed behaviour if pg_ident.conf is missing from an error to a warning 2014-10-22 Marco Nenciarini Remove code to replace output stream when quiet Previously the '-q' option was handled replacing the standard output stream with one which trows away averything it gets. Now it is not needed anymore because we haver a proper output module. 2014-09-26 Giulio Calacoci Remove all remaining output done by yield Migrate all the remaining part using yeld to do output to using the new output module. 2014-10-07 Marco Nenciarini Ignore fsync EINVAL errors on directories (#55) On some filesystem doing a fsync on a directory raises an EINVAL error. Ignoring it is usually safe. 2014-09-23 Giulio Calacoci Modified output module to access protected properties: quiet and debug 2014-09-10 Marco Nenciarini Fix bash autocompleter Minor changes: * Some code formatting adjustments Move cron retention policy management to a separate method 2014-09-05 Marco Nenciarini Fix dates in rpm changelog 2014-09-03 Giulio Calacoci Calculate backup WAL statistics only if the WALs are already processed 2014-09-02 Giulio Calacoci Change default LockFile behaviour to raise if fails acquisition 2014-09-01 Gabriele Bartolini Invoke WAL maintenance after a successful backup * At the end of the 'barman backup' command, maintenance operations are automatically started for successful backups (equivalent to manually executing a 'barman cron' command, just for that server) * Trashing of unuseful WALs (part of 'barman cron') has been changed as follows: * in case of one or more backups, delete WAL files older than the start WAL of the first backup * otherwise, trash WAL files in case of exclusive backup server (that is, not concurrent) 2014-09-03 Marco Nenciarini Remove redundant server argument from HookScriptRunner.env_from_wal_info() 2014-08-27 Gabriele Bartolini Add relpath() and fullpath() methods in WalInfoFile * Remove 'full_path' attribute in WalInfoFile * Add 'relpath()' method to WalInfoFile, which returns the relative path of a WAL file within the 'wals_directory' directory * Add 'fullpath()' method to WalInfoFile, which returns the full path of a WAL file within a server installation (requires a server object) 2014-08-23 Gabriele Bartolini Updated version in .spec file 2014-08-20 Marco Nenciarini Add build_config_from_dicts to testing_helpers module Make Config.Server, WalFileInfo and BackupInfo objects json encodable 2014-08-20 Gabriele Bartolini Added unit to JSON representation of a retention policy Started version 1.3.4-alpha.1 2014-08-18 Gabriele Bartolini Update the ChangeLog file Fixed typo in release date 2014-08-13 Gabriele Bartolini Prepared version 1.3.3 2014-08-12 Marco Nenciarini Add an unit-test for Server.get_wal_full_path() method 2014-08-12 Gabriele Bartolini Refactored building of full path of a WAL file 2014-08-01 Marco Nenciarini Report which file is about to be archived before actually doing it 2014-07-25 Giulio Calacoci Remove traceback from output when Barman is interrupted by CTRL-c Avoid flushing/fsyncing read only files Fixes: #49 EXCEPTION: [Errno 9] Bad file descriptor 2014-07-24 Giulio Calacoci Added Barman's version number to 'barman diagnose' 2014-07-22 Giulio Calacoci Move xlogdb_parse_line method in WalFileInfo class 2014-07-23 Marco Nenciarini Cleanup output API status at the end of test_output.py 2014-07-22 Gabriele Bartolini Estimates WAL production rate for a backup 2014-07-18 Giulio Calacoci Removed duplicate log message at the end of 'barman recover' wal segments copy Fix datetime.timedelta json serialization in 'barman diagnose' command 2014-07-17 Marco Nenciarini Update the ChangeLog file 2014-07-17 Gabriele Bartolini Prepared version 1.3.3-alpha.1 docs 2014-07-17 Marco Nenciarini Really fix "ssh" version detection in "barman diagnose" command 2014-07-16 Giulio Calacoci Add command line options for retry of backup/recover copy Implemented the --retry-times (including --no-retry) and --retry-sleep command line options for backup/recovery copy Emit warnings in case of unexptected configuration options 2014-07-14 Giulio Calacoci Reduce the verbosity of the log for "barman cron" Currently the "barman cron" command emits one log line for every WAL file that's archived (including the server name as a prefix). No log line is emitted for an empty cron run. Make recovery --target-time option more resilient to wrongly formatted values Workaround a bug in dateutil.parser.parse() implementation ref: https://bugs.launchpad.net/dateutil/+bug/1247643 Improved logging for "barman recover" command Default log prefix now contains barman process ID (pid) 2014-07-16 Marco Nenciarini Fix "ssh" version detection in "barman diagnose" command 2014-07-11 Marco Nenciarini Fix wrong variable name in BackupManager.delete_wal() 2014-07-09 Giulio Calacoci Add unit test for LockFile object and server.xlogdb() call Minor changes: - converted test_xlog.py to py.test style 2014-07-11 Giulio Calacoci Make sure remote WAL destination path is a directory Add a trailing slash to the remote WAL destination path, in order to ensure it is a directory 2014-07-07 Giulio Calacoci Fix serialisation of CvsOption during "barman diagnose" command 2014-07-11 Marco Nenciarini Use a WalFileInfo object when decoding an xlogdb line Add --no-human-readable to rsync --list-only invocation In rsync >= 3.1.0 the --list-only format changed adding digit groupings by default in "size" field. To obtain the pre 3.1.0 behavior you need to add --no-human-readable Ref: http://ftp.samba.org/pub/rsync/src/rsync-3.1.0-NEWS 2014-07-09 Marco Nenciarini Log any hook script failure with its output at warning level 2014-07-08 Gabriele Bartolini Wraps xlogdb() code in a try/finally block 2014-06-28 Marco Nenciarini Fix wait parameter logic in LockFile class In previous versions the wait argument on the LockFile constructor was mistakenly ignored, actually preventing the usage of a waiting lock through the Context Manager interface Always use utils.mkpath() to create directories that could already exist Minor changes: - In retry_backup_copy log the exception which caused the failure 2014-06-27 Marco Nenciarini Really ignore vanished files errors in rsync smart copy routine 2014-06-27 Gabriele Bartolini Added info messages for the four phases of the new rsync smart copy Minor changes: - Fix unit tests for basebackup_retry_* config values Updated documentation for 1.3.3-alpha1 Set default for basebackup_retry_times to 0 For compatibility with previous Barman versions, set basebackup_retry_times to 0 as default value. 2014-06-26 Giulio Calacoci Make sure timestamps are tz-aware anywhere in the code Minor changes: - Add basic unit tests for retention policies 2014-06-26 Marco Nenciarini Close all open descriptors but std{in,out,err} when spawning a child process Minor changes: - Remove some dead code - fix missing 'last_backup_maximum_age' as global option 2014-06-24 Gabriele Bartolini Display compression ratio for WALs in show-backup 2014-06-23 Giulio Calacoci Improved Nagios output for check command 2014-06-25 Giulio Calacoci Manage KeyboardInterrupt exception in 'barman backup' 2014-06-23 Gabriele Bartolini Added support for PostgreSQL 8.3 2014-06-24 Marco Nenciarini Updated rpm packaging spec to work with pre-releases Minor changes: - add rsync dependency to barman.spec file 2014-05-29 Giulio Calacoci Support for comma separated list options Added support for a new data type in configuration options: comma separated list values. The first option to be implemented is backup_options, now accepting a list of values. 2014-06-18 Marco Nenciarini Decode binary strings in command_wrapper This fixes python2.7 and python3 compatibility Minor changes: - make scripts/release.sh python3 compatible 2014-06-10 Giulio Calacoci Support for 'last_backup_max_age' This new global/server option allows administrators to set the max age of the last backup, making it easier to detect any issues with periodical backup execution. 2014-06-18 Marco Nenciarini Support for "smart" incremental recovery Avoid invoking rsync with --checksum option during recovery, while maintaining the same level of safety by splitting the copy operation in multiple steps. Barman will only use the --checksum option on files having identical time and size that have been modified after the start of the backup. This change greatly improves the speed of "incremental" recovery. Minor changes: - disable --checksum even for backup. During a backup the rsync destination directory is empty, so it is safe to go with a plain rsync - Put a ".barman-recover.info" with backup metadata inside the destination directory during recover. Use Postgres' server time for both begin_time and end_time Minor changes: - make sure exceptions during backup are logged with stacktraces - commit on disk the backup status just after issuing the PostgreSQL start_backup command Change version to 1.3.3-alpha.1 2014-06-09 Giulio Calacoci Added fsync() for backup and cron operations 2014-06-06 Marco Nenciarini Fix parsing of 'basebackup_retry_times' and 'basebackup_retry_sleep' options 2014-05-30 Giulio Calacoci Fix for #43 recovery.conf not copied on remote recovery 2014-05-08 Giulio Calacoci Retry option for base backup If a network error happens during rsync, add the ability to retry a defined number of time. Two options have been added: * basebackup_retry_times: INT (> 0, default 1) maximum number or retry before giving up * basebackup_retry_sleep: INT (> 0, default 10) wait time (seconds) before retrying, after an error 2014-05-29 Marco Nenciarini Improve robustness of backup code Improve error message about stop_backup failure 2014-04-23 Giulio Calacoci fixed missing pre/post archive parameters. #41 on sourceforge 2014-04-15 Marco Nenciarini Update the ChangeLog file Update unit tests to match current rsync flags 2014-04-15 Gabriele Bartolini Prepared source code for version 1.3.2 2014-04-15 Gabriele Bartolini Added checks for pg_extension (>= 9.1) and pg_is_in_recovery (>= 9.0) 2014-04-11 Marco Nenciarini Update the ChangeLog file 2014-04-10 Marco Nenciarini Always pass --checksum to rsync invocations Emit a warning if backup_options is set to an invalid value Clarify some "permission denied" error 2014-04-08 Gabriele Bartolini Cosmetic change: Pgespresso -> pgespresso 2014-04-07 Marco Nenciarini Update RPM spec file for 1.3.1 2014-04-04 Gabriele Bartolini Prepared documentation for version 1.3.1 2014-04-04 Marco Nenciarini Fix 'barman diagnose' python3 support Improved logging and error reporting 2014-04-03 Gabriele Bartolini Fixed SourceForge bug #36: Unhandled exception for minimum redundancy 2014-04-03 Giulio Calacoci Empty strings are now treated as None in Barman configuration 2014-04-02 Marco Nenciarini Removed spurious "file not found" message in cron output Add release information to 'barman diagnose' Sort 'barman show-server' output Use a Tablespace object to carry tablespace information 2014-03-26 Giulio Calacoci Protect during recovery tablespaces inside PGDATA * When performing a recovery operation, tablespaces that will be recovered inside the new destination directory (PGDATA) are be 'protected' by rsync. This avoids overwrites by rsync when copying PGDATA content. * Add debug messages to FS class 2014-03-24 Gabriele Bartolini Implementation of 'barman diagnose' command (JSON output) 2014-03-21 Gabriele Bartolini Concurrent backup using the 'pgespresso' extension * Fix bwlimit tablespaces backup (missing destination directory) * Purge unused wal files at first backup in concurrent mode * Exclusion of recovery.conf during backup 2014-03-19 Marco Nenciarini Fix unhandled exception in recover when destination dir is not writable 2014-02-19 Marco Nenciarini Make -q command line switch working again Also demote "another cron is running" message from error to info level. 2014-02-02 Marco Nenciarini Update the ChangeLog file Update RPM spec file for release 1.3.0 Review of NEWS and AUTHORS files 2014-01-31 Gabriele Bartolini Updated files for final release 2014-01-30 Marco Nenciarini Improve error messages during remote recovery 2014-01-29 Marco Nenciarini Use fsync to avoid xlog.db file corruption (Closes #32) Add network_compression configuration option (Closes #19) When network_compression is enabled, all network transfers are done using compression (if available). 2014-01-29 Gabriele Bartolini Check directories exist before executing a backup (#14) 2014-01-28 Giulio Calacoci Reduce log verbosity during initialisation phase 2014-01-28 Gabriele Bartolini Load configuration files after logger initialisation 2014-01-21 Marco Nenciarini Avoid tablespaces inside pgdata directory from being copied twice 2014-01-09 Marco Nenciarini Generalise recovery operations (local/remote) 2014-01-28 Gabriele Bartolini Reviewed documentation of WAL archive hook scripts 2014-01-07 Marco Nenciarini Add pre_archive_script and post_archive_scrip hook scripts 2014-01-23 Marco Nenciarini Refactor the LockFile management class to report permission errors. Fix 'Invalid cross-device link' error in cron when incoming is on a different filesystem (merge request #4 by Holger Hamann) 2014-01-22 Marco Nenciarini Port 'show-server' command to the new output interface 2014-01-21 Giulio Calacoci Updated copyright (2014) 2014-01-17 Marco Nenciarini Port 'status' and 'list-server' commands to the new output interface 2014-01-09 Marco Nenciarini Port the 'show-backup' command to the new output interface 2014-01-16 Giulio Calacoci Added implementation for backup command --immediate-checkpoint option and immediate_checkpoint configuration option 2014-01-08 Gabriele Bartolini Bump version number and add release notes for 1.3.0 2013-11-27 Giulio Calacoci Add unit tests for infofile and compression modules Fix some python3 compatibility bugs highlighted by the tests 2013-10-18 Marco Nenciarini Move barman._pretty_size() to barman.utils.pretty_size() 2014-01-03 Marco Nenciarini Implement BackupInfo as a FieldListFile and move it in infofile module. 2014-01-07 Marco Nenciarini Refactor output to a dedicate module. The following commands have been ported to the new interface: * backup * check * list-backup A special NagiosOutputWriter has been added to support Nagios compatible output for the check command WARNING: this code doesn't run due to a circular dependency. The issue will be fixed in the next commit 2013-09-12 Marco Nenciarini Isolate subrocesses' stdin/stdout in command_wrappers module 2014-01-07 Marco Nenciarini Refactor hooks management 2013-09-12 Marco Nenciarini Split out logging configuration and userid enforcement from the configuration class. 2013-12-16 Gabriele Bartolini Added rebuild-xlogdb command man page 2013-11-08 Marco Nenciarini Implement the rebuild-xlogdb command. (Closes #27) 2013-11-19 Giulio Calacoci added documentation for tablespaces relocation (#22) 2013-10-30 Gabriele Bartolini Added TODO list 2013-09-05 Marco Nenciarini Update the ChangeLog file Bump version to 1.2.3 2013-08-29 Gabriele Bartolini Updated README and man page Added stub of release notes 2013-08-26 Marco Nenciarini Initial Python 3 support Update setup.py to support py.test and recent setuptools 2013-08-24 Damon Snyder 27: Addresses potential corruption of WAL xlog.db files. In barman.lockfile.release() the file is unlinked (deleted). This effectively nullifies any future attempts to lock the file by a blocking process by deleting the open file table entry upon which the flock is based. This commit removes the unlink and instead unlocks the file and then closes the file descriptor leaving the lock file and open file table entry intact. 2013-08-22 Marco Nenciarini Add support for restore target name (PostgreSQL 9.1+) 2013-08-21 Marco Nenciarini PostgreSQL version in backup.info file is an integer Make WAL sequence calculation compatible with PostgreSQL 9.3 With PostgreSQL 9.3 WAL files are written in a continuous stream, rather than skipping the last 16MB segment every 4GB, meaning WAL filenames may end in FF. 2013-06-24 Marco Nenciarini Update the ChangeLog file Fix config file parser tests Bump version to 1.2.2 Fix python 2.6 compatibility Fix history in spec file 2013-06-17 Marco Nenciarini Update RPM spec file 2013-06-13 Marco Nenciarini Update the ChangeLog file Fix remote recovery with bwlimit on a tablespace 2013-06-07 Marco Nenciarini Added the "tablespace_bandwidth_limit" option 2013-06-12 Gabriele Bartolini Updated docs and man pages for 1.2.1 Prepared NEWS file for 1.2.1 release 2013-04-26 Gabriele Bartolini Added the "bandwidth_limit" global/server option which allows to limit the I/O bandwidth (in KBPS) for backup and recovery operations Added /etc/barman/barman.conf as default location 2013-03-13 Gabriele Bartolini Removed duplicate message for previous backup in show command 2013-03-07 Gabriele Bartolini Cosmetic change in message for "all" reserved section 2013-02-08 Marco Nenciarini Avoid triggering the minimum_redundancy check on FAILED backups Add BARMAN_VERSION to hook script environment 2013-01-31 Marco Nenciarini Update the ChangeLog file Update RPM's spec files 2013-01-30 Gabriele Bartolini Finalised files for version 1.2.0 2013-01-28 Marco Nenciarini Forbid the usage of 'all' word as server name 2013-01-11 Gabriele Bartolini Added basic support for Nagios plugin output for check command through the --nagios option 2013-01-28 Marco Nenciarini Add @expects_obj decorator to cli function as required by the upcoming Argh 1.0 API 2013-01-11 Marco Nenciarini Migratte to new argh api. Now barman requires arg => 0.21.2 and argcomplete- 2013-01-11 Gabriele Bartolini Prepared release notes 2012-12-18 Marco Nenciarini Fix typo in doc/barman.conf 2012-12-14 Marco Nenciarini Return failure exit code if backup command fails in any way 2012-12-14 Gabriele Bartolini Prepared copyright lines for 2013 Updated documentation and man pages Added retention policy examples in configuration file 2012-12-13 Marco Nenciarini Q/A on retention policy code 2012-12-12 Marco Nenciarini Fix configuration parser unit tests Exit with error if an invalid server name is passed in any command which takes a list of server 2012-12-08 Gabriele Bartolini Add retention status to show-backup and list-backup commands Auto-management of retention policies for base backups Using the report() method for retention policies, enforce retention policy through cron (if policy mode is 'auto'), by deleting OBSOLETE backups. Retention status and report() method for retention policies Created the following states for retention policies: VALID, OBSOLETE, NONE and POTENTIALLY_OBSOLETE (an object which is OBSOLETE but cannot be removed automatically due to minimum_redundancy requirements). Created the report() method for the retention policy base class, which exected the _backup_report() method for base backups and the _wal_report() method for WAL retention policies (currently not enforced). The report method iterates through the DONE backups and according to the retention policy, classifies the backup. RedundancyRetentionPolicy uses the number of backups, RecoveryWindowRetentionPolicy uses the time window and the recoverability point concept. Integrated minimum_redundancy with "barman check" Initialisation of retention policies for a server Added the _init_retention_policies() method in the Server class constructor, which integrates with the new RetentionPolicy classes and performs syntax checking. Integrated retention policies with log, 'barman check' and 'barman status'. String representation conforms to retention syntax The string representation produces now a syntax-valid retention policy configuration string. The previous __str__ method has been renamed into debug() SimpleWALRetentionPolicy objects are now created from the server's main retention policy by the factory class. 2012-12-07 Gabriele Bartolini Add the global/server option minimum_redundancy. Check it is >= 0. Guarantees that when delete is performed (or retention policies are enforced), this is the minimum number of backups to be kept for that server. Add support for retention_policy_mode global/server option which defines the method for enforcing retention policies (currently only "auto", in future versions "manual" will be allowed) Added first stub of retention policy classes Started version 1.2.0 2012-12-04 Marco Nenciarini Fix unit config tests Update the ChangeLog file Add ssl_*_file and unix_socket_directory to dangerous options list Display tablespace's oid in show-backup output Alphabetically sort servers in all commands output Don't give up on first error in 'barman check all' command 2012-12-03 Gabriele Bartolini Added sorting of files in configuration directory 2012-11-29 Marco Nenciarini Fix regression in barman check command when configuration_files_directory is None Update rpm files to 1.1.2 release 2012-11-29 Carlo Ascani Update README 2012-11-29 Gabriele Bartolini Prepared files for release 2012-11-28 Gabriele Bartolini Add the configuration_files_directory option which allows to include multiple files from a directory 2012-11-29 Carlo Ascani Update README 2012-11-28 Marco Nenciarini Update NEWS file 2012-11-05 Gabriele Bartolini Added support for list-backup all 2012-11-04 Gabriele Bartolini Added latest/oldest for show-backup, delete, list-files and recover commands Added get_first_backup and get_last_backup functions to Server class Added application_name management for PostgreSQL >= 9.0 2012-11-13 Gabriele Bartolini Switched to version 1.1.2 Continue if a WAL file is not found during delete (bug #18) 2012-11-04 Gabriele Bartolini Includes version 90200 for tablespace new function 2012-10-16 Marco Nenciarini Update the ChangeLog file Update NEWS file and rpm package Bump version to 1.1.1 Add more information about the failing line in xlogdb_parse_line errors 2012-10-15 Marco Nenciarini Fix two bug on recover command 2012-10-12 Marco Nenciarini Update the ChangeLog file Update rpm changelog Make recover fail if an invalid tablespace relocation rule is given Remove unused imports from cli.py 2012-10-11 Gabriele Bartolini Updated version to 1.1.0 Fixes bug #12 2012-10-11 Marco Nenciarini Fail fast on recover command if the destination directory contains the ':' character (Closes: #4) Fix typo in recovery messages Report an informative message when pg_start_backup() invocation fails because an exclusive backup is already running (Closes: #8) Make current_action an attribute of BackupManager class 2012-10-08 Gabriele Bartolini Added ticket #10 to NEWS Add pg_config_detect_possible_issues function for issue #10 2012-10-04 Gabriele Bartolini Updated NEWS file with bug fixing #9 Fixes issue #9 on pg_tablespace_location() for 9.2 2012-08-31 Marco Nenciarini Add BARMAN_PREVIOUS_ID variable to hooks environment 2012-08-20 Marco Nenciarini Merge spec changes from Devrim Add BARMAN_ERROR and BARMAN_STATUS variables to hook's environment Added backup all documentation to README 2012-08-20 Gabriele Bartolini Updated release notes Set version to 1.0.1 2012-08-20 Marco Nenciarini Document {pre,post}_backup_script in README Document {pre,post}_backup_script in configuration man-page 2012-08-17 Marco Nenciarini Add pre/post backup hook scripts definition (Closes: #7) Add the possibility to manage hook scripts before and after a base backup. Add the global (overridden per server) configuration options called: * pre_backup_script: executed before a backup * post_backup_script: executed after a backup Use the environment to pass at least the following variabiles: * BARMAN_BACKUP_DIR: backup destination directory * BARMAN_BACKUP_ID: ID of the backup * BARMAN_CONFIGURATION: configuration file used by barman * BARMAN_PHASE: 'pre' or 'post' * BARMAN_SERVER: name of the server The script definition is passed to the shell and can return any exit code. Barman won't perform any exit code check. It will simply log the result in the log file. To test it you can try adding pre_backup_script = env | grep ^BARMAN post_backup_script = env | grep ^BARMAN in your barman config and you'll see the variables on console. 2012-08-16 Marco Nenciarini Add documentation for 'backup all' command. 2012-07-19 Gabriele Bartolini Add 'backup all' shortcut and, in general, multiple servers specification (issue #1) Add 'backup all' shortcut and, in general, multiple servers specification (issue #1) 2012-07-16 Gabriele Bartolini Fixed typo (thanks to Daymel Bonne Solís) 2012-07-06 Marco Nenciarini Initial commit barman-2.3/doc/0000755000076500000240000000000013153300776014125 5ustar mnenciastaff00000000000000barman-2.3/doc/barman.10000644000076500000240000004025213153300354015442 0ustar mnenciastaff00000000000000.\" Automatically generated by Pandoc 1.19.2.1 .\" .TH "BARMAN" "1" "September 05, 2017" "Barman User manuals" "Version 2.3" .hy .SH NAME .PP barman \- Backup and Recovery Manager for PostgreSQL .SH SYNOPSIS .PP barman [\f[I]OPTIONS\f[]] \f[I]COMMAND\f[] .SH DESCRIPTION .PP Barman is an administration tool for disaster recovery of PostgreSQL servers written in Python and maintained by 2ndQuadrant. Barman can perform remote backups of multiple servers in business critical environments and helps DBAs during the recovery phase. .SH OPTIONS .TP .B \-v, \-\-version Show program version number and exit. .RS .RE .TP .B \-q, \-\-quiet Do not output anything. Useful for cron scripts. .RS .RE .TP .B \-h, \-\-help Show a help message and exit. .RS .RE .TP .B \-c \f[I]CONFIG\f[], \-\-config \f[I]CONFIG\f[] Use the specified configuration file. .RS .RE .SH COMMANDS .PP Important: every command has a help option .TP .B archive\-wal \f[I]SERVER_NAME\f[] Get any incoming xlog file (both through standard \f[C]archive_command\f[] and streaming replication, where applicable) and moves them in the WAL archive for that server. If necessary, apply compression when requested by the user. .RS .RE .TP .B cron Perform maintenance tasks, such as enforcing retention policies or WAL files management. .RS .RE .TP .B list\-server Show all the configured servers, and their descriptions. .RS .RE .TP .B show\-server \f[I]SERVER_NAME\f[] Show information about \f[C]SERVER_NAME\f[], including: \f[C]conninfo\f[], \f[C]backup_directory\f[], \f[C]wals_directory\f[] and many more. Specify \f[C]all\f[] as \f[C]SERVER_NAME\f[] to show information about all the configured servers. .RS .RE .TP .B status \f[I]SERVER_NAME\f[] Show information about the status of a server, including: number of available backups, \f[C]archive_command\f[], \f[C]archive_status\f[] and many more. For example: .RS .RE .IP .nf \f[C] Server\ quagmire: \ \ Description:\ The\ Giggity\ database \ \ Passive\ node:\ False \ \ PostgreSQL\ version:\ 9.3.9 \ \ pgespresso\ extension:\ Not\ available \ \ PostgreSQL\ Data\ directory:\ /srv/postgresql/9.3/data \ \ PostgreSQL\ \[aq]archive_command\[aq]\ setting:\ rsync\ \-a\ %p\ barman\@backup:/var/lib/barman/quagmire/incoming \ \ Last\ archived\ WAL:\ 0000000100003103000000AD \ \ Current\ WAL\ segment:\ 0000000100003103000000AE \ \ Retention\ policies:\ enforced\ (mode:\ auto,\ retention:\ REDUNDANCY\ 2,\ WAL\ retention:\ MAIN) \ \ No.\ of\ available\ backups:\ 2 \ \ First\ available\ backup:\ 20150908T003001 \ \ Last\ available\ backup:\ 20150909T003001 \ \ Minimum\ redundancy\ requirements:\ satisfied\ (2/1) \f[] .fi .TP .B check \f[I]SERVER_NAME\f[] Show diagnostic information about \f[C]SERVER_NAME\f[], including: Ssh connection check, PostgreSQL version, configuration and backup directories, archiving process, streaming process, replication slots, etc. Specify \f[C]all\f[] as \f[C]SERVER_NAME\f[] to show diagnostic information about all the configured servers. .RS .TP .B \-\-nagios Nagios plugin compatible output .RS .RE .RE .TP .B diagnose Collect diagnostic information about the server where barman is installed and all the configured servers, including: global configuration, SSH version, Python version, \f[C]rsync\f[] version, as well as current configuration and status of all servers. .RS .RE .TP .B backup \f[I]SERVER_NAME\f[] Perform a backup of \f[C]SERVER_NAME\f[] using parameters specified in the configuration file. Specify \f[C]all\f[] as \f[C]SERVER_NAME\f[] to perform a backup of all the configured servers. .RS .TP .B \-\-immediate\-checkpoint forces the initial checkpoint to be done as quickly as possible. Overrides value of the parameter \f[C]immediate_checkpoint\f[], if present in the configuration file. .RS .RE .TP .B \-\-no\-immediate\-checkpoint forces to wait for the checkpoint. Overrides value of the parameter \f[C]immediate_checkpoint\f[], if present in the configuration file. .RS .RE .TP .B \-\-reuse\-backup [INCREMENTAL_TYPE] Overrides \f[C]reuse_backup\f[] option behaviour. Possible values for \f[C]INCREMENTAL_TYPE\f[] are: .RS .IP \[bu] 2 \f[I]off\f[]: do not reuse the last available backup; .IP \[bu] 2 \f[I]copy\f[]: reuse the last available backup for a server and create a copy of the unchanged files (reduce backup time); .IP \[bu] 2 \f[I]link\f[]: reuse the last available backup for a server and create a hard link of the unchanged files (reduce backup time and space); .PP \f[C]link\f[] is the default target if \f[C]\-\-reuse\-backup\f[] is used and \f[C]INCREMENTAL_TYPE\f[] is not explicited. .RE .TP .B \-\-retry\-times Number of retries of base backup copy, after an error. Used during both backup and recovery operations. Overrides value of the parameter \f[C]basebackup_retry_times\f[], if present in the configuration file. .RS .RE .TP .B \-\-no\-retry Same as \f[C]\-\-retry\-times\ 0\f[] .RS .RE .TP .B \-\-retry\-sleep Number of seconds of wait after a failed copy, before retrying. Used during both backup and recovery operations. Overrides value of the parameter \f[C]basebackup_retry_sleep\f[], if present in the configuration file. .RS .RE .TP .B \-j , \-\-jobs Number of parallel workers to copy files during backup. Overrides value of the parameter \f[C]parallel_jobs\f[], if present in the configuration file. .RS .RE .RE .TP .B list\-backup \f[I]SERVER_NAME\f[] Show available backups for \f[C]SERVER_NAME\f[]. This command is useful to retrieve a backup ID. For example: .RS .RE .IP .nf \f[C] servername\ 20111104T102647\ \-\ Fri\ Nov\ \ 4\ 10:26:48\ 2011\ \-\ Size:\ 17.0\ MiB\ \-\ WAL\ Size:\ 100\ B \f[] .fi .IP .nf \f[C] In\ this\ case,\ *20111104T102647*\ is\ the\ backup\ ID. \f[] .fi .TP .B show\-backup \f[I]SERVER_NAME\f[] \f[I]BACKUP_ID\f[] Show detailed information about a particular backup, identified by the server name and the backup ID. See the Backup ID shortcuts (#shortcuts) section below for available shortcuts. For example: .RS .RE .IP .nf \f[C] Backup\ 20150828T130001: \ \ Server\ Name\ \ \ \ \ \ \ \ \ \ \ \ :\ quagmire \ \ Status\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ :\ DONE \ \ PostgreSQL\ Version\ \ \ \ \ :\ 90402 \ \ PGDATA\ directory\ \ \ \ \ \ \ :\ /srv/postgresql/9.4/main/data \ \ Base\ backup\ information: \ \ \ \ Disk\ usage\ \ \ \ \ \ \ \ \ \ \ :\ 12.4\ TiB\ (12.4\ TiB\ with\ WALs) \ \ \ \ Incremental\ size\ \ \ \ \ :\ 4.9\ TiB\ (\-60.02%) \ \ \ \ Timeline\ \ \ \ \ \ \ \ \ \ \ \ \ :\ 1 \ \ \ \ Begin\ WAL\ \ \ \ \ \ \ \ \ \ \ \ :\ 0000000100000CFD000000AD \ \ \ \ End\ WAL\ \ \ \ \ \ \ \ \ \ \ \ \ \ :\ 0000000100000D0D00000008 \ \ \ \ WAL\ number\ \ \ \ \ \ \ \ \ \ \ :\ 3932 \ \ \ \ WAL\ compression\ ratio:\ 79.51% \ \ \ \ Begin\ time\ \ \ \ \ \ \ \ \ \ \ :\ 2015\-08\-28\ 13:00:01.633925+00:00 \ \ \ \ End\ time\ \ \ \ \ \ \ \ \ \ \ \ \ :\ 2015\-08\-29\ 10:27:06.522846+00:00 \ \ \ \ Begin\ Offset\ \ \ \ \ \ \ \ \ :\ 1575048 \ \ \ \ End\ Offset\ \ \ \ \ \ \ \ \ \ \ :\ 13853016 \ \ \ \ Begin\ XLOG\ \ \ \ \ \ \ \ \ \ \ :\ CFD/AD180888 \ \ \ \ End\ XLOG\ \ \ \ \ \ \ \ \ \ \ \ \ :\ D0D/8D36158 \ \ WAL\ information: \ \ \ \ No\ of\ files\ \ \ \ \ \ \ \ \ \ :\ 35039 \ \ \ \ Disk\ usage\ \ \ \ \ \ \ \ \ \ \ :\ 121.5\ GiB \ \ \ \ WAL\ rate\ \ \ \ \ \ \ \ \ \ \ \ \ :\ 275.50/hour \ \ \ \ Compression\ ratio\ \ \ \ :\ 77.81% \ \ \ \ Last\ available\ \ \ \ \ \ \ :\ 0000000100000D95000000E7 \ \ Catalog\ information: \ \ \ \ Retention\ Policy\ \ \ \ \ :\ not\ enforced \ \ \ \ Previous\ Backup\ \ \ \ \ \ :\ 20150821T130001 \ \ \ \ Next\ Backup\ \ \ \ \ \ \ \ \ \ :\ \-\ (this\ is\ the\ latest\ base\ backup) \f[] .fi .TP .B list\-files \f[I][OPTIONS]\f[] \f[I]SERVER_NAME\f[] \f[I]BACKUP_ID\f[] List all the files in a particular backup, identified by the server name and the backup ID. See the Backup ID shortcuts (#shortcuts) section below for available shortcuts. .RS .TP .B \-\-target \f[I]TARGET_TYPE\f[] Possible values for TARGET_TYPE are: .RS .IP \[bu] 2 \f[I]data\f[]: lists just the data files; .IP \[bu] 2 \f[I]standalone\f[]: lists the base backup files, including required WAL files; .IP \[bu] 2 \f[I]wal\f[]: lists all the WAL files between the start of the base backup and the end of the log / the start of the following base backup (depending on whether the specified base backup is the most recent one available); .IP \[bu] 2 \f[I]full\f[]: same as data + wal. .PP The default value is \f[C]standalone\f[]. .RE .RE .TP .B rebuild\-xlogdb \f[I]SERVER_NAME\f[] Perform a rebuild of the WAL file metadata for \f[C]SERVER_NAME\f[] (or every server, using the \f[C]all\f[] shortcut) guessing it from the disk content. The metadata of the WAL archive is contained in the \f[C]xlog.db\f[] file, and every Barman server has its own copy. .RS .RE .TP .B recover \f[I][OPTIONS]\f[] \f[I]SERVER_NAME\f[] \f[I]BACKUP_ID\f[] \f[I]DESTINATION_DIRECTORY\f[] Recover a backup in a given directory (local or remote, depending on the \f[C]\-\-remote\-ssh\-command\f[] option settings). See the Backup ID shortcuts (#shortcuts) section below for available shortcuts. .RS .TP .B \-\-target\-tli \f[I]TARGET_TLI\f[] Recover the specified timeline. .RS .RE .TP .B \-\-target\-time \f[I]TARGET_TIME\f[] Recover to the specified time. .RS .PP You can use any valid unambiguous representation (e.g: "YYYY\-MM\-DD HH:MM:SS.mmm"). .RE .TP .B \-\-target\-xid \f[I]TARGET_XID\f[] Recover to the specified transaction ID. .RS .RE .TP .B \-\-target\-name \f[I]TARGET_NAME\f[] Recover to the named restore point previously created with the \f[C]pg_create_restore_point(name)\f[] (for PostgreSQL 9.1 and above users). .RS .RE .TP .B \-\-target\-immediate Recover ends when a consistent state is reached (end of the base backup) .RS .RE .TP .B \-\-exclusive Set target xid to be non inclusive. .RS .RE .TP .B \-\-tablespace \f[I]NAME:LOCATION\f[] Specify tablespace relocation rule. .RS .RE .TP .B \-\-remote\-ssh\-command \f[I]SSH_COMMAND\f[] This options activates remote recovery, by specifying the secure shell command to be launched on a remote host. This is the equivalent of the "ssh_command" server option in the configuration file for remote recovery. Example: \[aq]ssh postgres\@db2\[aq]. .RS .RE .TP .B \-\-retry\-times \f[I]RETRY_TIMES\f[] Number of retries of data copy during base backup after an error. Overrides value of the parameter \f[C]basebackup_retry_times\f[], if present in the configuration file. .RS .RE .TP .B \-\-no\-retry Same as \f[C]\-\-retry\-times\ 0\f[] .RS .RE .TP .B \-\-retry\-sleep Number of seconds of wait after a failed copy, before retrying. Overrides value of the parameter \f[C]basebackup_retry_sleep\f[], if present in the configuration file. .RS .RE .TP .B \-j , \-\-jobs Number of parallel workers to copy files during recovery. Overrides value of the parameter \f[C]parallel_jobs\f[], if present in the configuration file. Works only for servers configured through \f[C]rsync\f[]/SSH. .RS .RE .TP .B \-\-get\-wal, \-\-no\-get\-wal Enable/Disable usage of \f[C]get\-wal\f[] for WAL fetching during recovery. Default is based on \f[C]recovery_options\f[] setting. .RS .RE .TP .B \-\-network\-compression, \-\-no\-network\-compression Enable/Disable network compression during remote recovery. Default is based on \f[C]network_compression\f[] configuration setting. .RS .RE .RE .TP .B get\-wal \f[I][OPTIONS]\f[] \f[I]SERVER_NAME\f[] \f[I]WAL_ID\f[] Retrieve a WAL file from the \f[C]xlog\f[] archive of a given server. By default, the requested WAL file, if found, is returned as uncompressed content to \f[C]STDOUT\f[]. The following options allow users to change this behaviour: .RS .TP .B \-o \f[I]OUTPUT_DIRECTORY\f[] destination directory where the \f[C]get\-wal\f[] will deposit the requested WAL .RS .RE .TP .B \-z output will be compressed using gzip .RS .RE .TP .B \-j output will be compressed using bzip2 .RS .RE .TP .B \-p \f[I]SIZE\f[] peek from the WAL archive up to \f[I]SIZE\f[] WAL files, starting from the requested one. \[aq]SIZE\[aq] must be an integer >= 1. When invoked with this option, get\-wal returns a list of zero to \[aq]SIZE\[aq] WAL segment names, one per row. .RS .RE .RE .TP .B switch\-wal \f[I]SERVER_NAME\f[] Execute pg_switch_wal() on the target server (from PostgreSQL 10), or pg_switch_xlog (for PostgreSQL 8.3 to 9.6). .RS .TP .B \-\-force Forces the switch by executing CHECKPOINT before pg_switch_xlog(). \f[I]IMPORTANT:\f[] executing a CHECKPOINT might increase I/O load on a PostgreSQL server. Use this option with care. .RS .RE .TP .B \-\-archive Wait for one xlog file to be archived. If after a defined amount of time (default: 30 seconds) no xlog file is archived, Barman will teminate with failure exit code. .RS .RE .TP .B \-\-archive\-timeout \f[I]TIMEOUT\f[] Specifies the amount of time in seconds (default: 30 seconds) the archiver will wait for a new xlog file to be archived before timing out. .RS .RE .RE .TP .B switch\-xlog \f[I]SERVER_NAME\f[] Alias for switch\-wal (kept for back\-compatibility) .RS .RE .TP .B receive\-wal \f[I]SERVER_NAME\f[] Start the stream of transaction logs for a server. The process relies on \f[C]pg_receivexlog\f[] to receive WAL files from the PostgreSQL servers through the streaming protocol. .RS .TP .B \-\-stop stop the receive\-wal process for the server .RS .RE .TP .B \-\-reset reset the status of receive\-wal, restarting the streaming from the current WAL file of the server .RS .RE .TP .B \-\-create\-slot create the physical replication slot configured with the \f[C]slot_name\f[] configuration parameter .RS .RE .TP .B \-\-drop\-slot drop the physical replication slot configured with the \f[C]slot_name\f[] configuration parameter .RS .RE .RE .TP .B delete \f[I]SERVER_NAME\f[] \f[I]BACKUP_ID\f[] Delete the specified backup. Backup ID shortcuts (#shortcuts) section below for available shortcuts. .RS .RE .TP .B replication\-status \f[I][OPTIONS]\f[] \f[I]SERVER_NAME\f[] Shows live information and status of any streaming client attached to the given server (or servers). Default behaviour can be changed through the following options: .RS .TP .B \-\-minimal machine readable output (default: False) .RS .RE .TP .B \-\-target \f[I]TARGET_TYPE\f[] Possible values for TARGET_TYPE are: .RS .IP \[bu] 2 \f[I]hot\-standby\f[]: lists only hot standby servers .IP \[bu] 2 \f[I]wal\-streamer\f[]: lists only WAL streaming clients, such as pg_receivexlog .IP \[bu] 2 \f[I]all\f[]: any streaming client (default) .RE .RE .SH BACKUP ID SHORTCUTS .PP Rather than using the timestamp backup ID, you can use any of the following shortcuts/aliases to identity a backup for a given server: .TP .B first Oldest available backup for that server, in chronological order. .RS .RE .TP .B last Latest available backup for that server, in chronological order. .RS .RE .TP .B latest same ast \f[I]last\f[]. .RS .RE .TP .B oldest same ast \f[I]first\f[]. .RS .RE .SH EXIT STATUS .TP .B 0 Success .RS .RE .TP .B Not zero Failure .RS .RE .SH SEE ALSO .PP \f[C]barman\f[] (5). .SH BUGS .PP Barman has been extensively tested, and is currently being used in several production environments. However, we cannot exclude the presence of bugs. .PP Any bug can be reported via the Sourceforge bug tracker. Along the bug submission, users can provide developers with diagnostics information obtained through the \f[C]barman\ diagnose\f[] command. .SH AUTHORS .PP In alphabetical order: .IP \[bu] 2 Gabriele Bartolini (architect) .IP \[bu] 2 Jonathan Battiato (QA/testing) .IP \[bu] 2 Giulio Calacoci (developer) .IP \[bu] 2 Francesco Canovai (QA/testing) .IP \[bu] 2 Leonardo Cecchi (developer) .IP \[bu] 2 Gianni Ciolli (QA/testing) .IP \[bu] 2 Britt Cole (documentation) .IP \[bu] 2 Marco Nenciarini (project leader) .IP \[bu] 2 Rubens Souza (QA/testing) .PP Past contributors: .IP \[bu] 2 Carlo Ascani .IP \[bu] 2 Stefano Bianucci .IP \[bu] 2 Giuseppe Broccolo .SH RESOURCES .IP \[bu] 2 Homepage: .IP \[bu] 2 Documentation: .IP \[bu] 2 Professional support: .SH COPYING .PP Barman is the property of 2ndQuadrant Limited and its code is distributed under GNU General Public License v3. .PP Copyright (C) 2011\-2017 2ndQuadrant Limited \- . .SH AUTHORS 2ndQuadrant Limited . barman-2.3/doc/barman.1.md0000644000076500000240000003525713153300354016052 0ustar mnenciastaff00000000000000% BARMAN(1) Barman User manuals | Version 2.3 % 2ndQuadrant Limited % September 05, 2017 # NAME barman - Backup and Recovery Manager for PostgreSQL # SYNOPSIS barman [*OPTIONS*] *COMMAND* # DESCRIPTION Barman is an administration tool for disaster recovery of PostgreSQL servers written in Python and maintained by 2ndQuadrant. Barman can perform remote backups of multiple servers in business critical environments and helps DBAs during the recovery phase. # OPTIONS -v, --version : Show program version number and exit. -q, --quiet : Do not output anything. Useful for cron scripts. -h, --help : Show a help message and exit. -c *CONFIG*, --config *CONFIG* : Use the specified configuration file. # COMMANDS Important: every command has a help option archive-wal *SERVER_NAME* : Get any incoming xlog file (both through standard `archive_command` and streaming replication, where applicable) and moves them in the WAL archive for that server. If necessary, apply compression when requested by the user. cron : Perform maintenance tasks, such as enforcing retention policies or WAL files management. list-server : Show all the configured servers, and their descriptions. show-server *SERVER_NAME* : Show information about `SERVER_NAME`, including: `conninfo`, `backup_directory`, `wals_directory` and many more. Specify `all` as `SERVER_NAME` to show information about all the configured servers. status *SERVER_NAME* : Show information about the status of a server, including: number of available backups, `archive_command`, `archive_status` and many more. For example: ``` Server quagmire: Description: The Giggity database Passive node: False PostgreSQL version: 9.3.9 pgespresso extension: Not available PostgreSQL Data directory: /srv/postgresql/9.3/data PostgreSQL 'archive_command' setting: rsync -a %p barman@backup:/var/lib/barman/quagmire/incoming Last archived WAL: 0000000100003103000000AD Current WAL segment: 0000000100003103000000AE Retention policies: enforced (mode: auto, retention: REDUNDANCY 2, WAL retention: MAIN) No. of available backups: 2 First available backup: 20150908T003001 Last available backup: 20150909T003001 Minimum redundancy requirements: satisfied (2/1) ``` check *SERVER_NAME* : Show diagnostic information about `SERVER_NAME`, including: Ssh connection check, PostgreSQL version, configuration and backup directories, archiving process, streaming process, replication slots, etc. Specify `all` as `SERVER_NAME` to show diagnostic information about all the configured servers. --nagios : Nagios plugin compatible output diagnose : Collect diagnostic information about the server where barman is installed and all the configured servers, including: global configuration, SSH version, Python version, `rsync` version, as well as current configuration and status of all servers. backup *SERVER_NAME* : Perform a backup of `SERVER_NAME` using parameters specified in the configuration file. Specify `all` as `SERVER_NAME` to perform a backup of all the configured servers. --immediate-checkpoint : forces the initial checkpoint to be done as quickly as possible. Overrides value of the parameter `immediate_checkpoint`, if present in the configuration file. --no-immediate-checkpoint : forces to wait for the checkpoint. Overrides value of the parameter `immediate_checkpoint`, if present in the configuration file. --reuse-backup [INCREMENTAL_TYPE] : Overrides `reuse_backup` option behaviour. Possible values for `INCREMENTAL_TYPE` are: - *off*: do not reuse the last available backup; - *copy*: reuse the last available backup for a server and create a copy of the unchanged files (reduce backup time); - *link*: reuse the last available backup for a server and create a hard link of the unchanged files (reduce backup time and space); `link` is the default target if `--reuse-backup` is used and `INCREMENTAL_TYPE` is not explicited. --retry-times : Number of retries of base backup copy, after an error. Used during both backup and recovery operations. Overrides value of the parameter `basebackup_retry_times`, if present in the configuration file. --no-retry : Same as `--retry-times 0` --retry-sleep : Number of seconds of wait after a failed copy, before retrying. Used during both backup and recovery operations. Overrides value of the parameter `basebackup_retry_sleep`, if present in the configuration file. -j , --jobs : Number of parallel workers to copy files during backup. Overrides value of the parameter `parallel_jobs`, if present in the configuration file. list-backup *SERVER_NAME* : Show available backups for `SERVER_NAME`. This command is useful to retrieve a backup ID. For example: ``` servername 20111104T102647 - Fri Nov 4 10:26:48 2011 - Size: 17.0 MiB - WAL Size: 100 B ``` In this case, *20111104T102647* is the backup ID. show-backup *SERVER_NAME* *BACKUP_ID* : Show detailed information about a particular backup, identified by the server name and the backup ID. See the [Backup ID shortcuts](#shortcuts) section below for available shortcuts. For example: ``` Backup 20150828T130001: Server Name : quagmire Status : DONE PostgreSQL Version : 90402 PGDATA directory : /srv/postgresql/9.4/main/data Base backup information: Disk usage : 12.4 TiB (12.4 TiB with WALs) Incremental size : 4.9 TiB (-60.02%) Timeline : 1 Begin WAL : 0000000100000CFD000000AD End WAL : 0000000100000D0D00000008 WAL number : 3932 WAL compression ratio: 79.51% Begin time : 2015-08-28 13:00:01.633925+00:00 End time : 2015-08-29 10:27:06.522846+00:00 Begin Offset : 1575048 End Offset : 13853016 Begin XLOG : CFD/AD180888 End XLOG : D0D/8D36158 WAL information: No of files : 35039 Disk usage : 121.5 GiB WAL rate : 275.50/hour Compression ratio : 77.81% Last available : 0000000100000D95000000E7 Catalog information: Retention Policy : not enforced Previous Backup : 20150821T130001 Next Backup : - (this is the latest base backup) ``` list-files *\[OPTIONS\]* *SERVER_NAME* *BACKUP_ID* : List all the files in a particular backup, identified by the server name and the backup ID. See the [Backup ID shortcuts](#shortcuts) section below for available shortcuts. --target *TARGET_TYPE* : Possible values for TARGET_TYPE are: - *data*: lists just the data files; - *standalone*: lists the base backup files, including required WAL files; - *wal*: lists all the WAL files between the start of the base backup and the end of the log / the start of the following base backup (depending on whether the specified base backup is the most recent one available); - *full*: same as data + wal. The default value is `standalone`. rebuild-xlogdb *SERVER_NAME* : Perform a rebuild of the WAL file metadata for `SERVER_NAME` (or every server, using the `all` shortcut) guessing it from the disk content. The metadata of the WAL archive is contained in the `xlog.db` file, and every Barman server has its own copy. recover *\[OPTIONS\]* *SERVER_NAME* *BACKUP_ID* *DESTINATION_DIRECTORY* : Recover a backup in a given directory (local or remote, depending on the `--remote-ssh-command` option settings). See the [Backup ID shortcuts](#shortcuts) section below for available shortcuts. --target-tli *TARGET_TLI* : Recover the specified timeline. --target-time *TARGET_TIME* : Recover to the specified time. You can use any valid unambiguous representation (e.g: "YYYY-MM-DD HH:MM:SS.mmm"). --target-xid *TARGET_XID* : Recover to the specified transaction ID. --target-name *TARGET_NAME* : Recover to the named restore point previously created with the `pg_create_restore_point(name)` (for PostgreSQL 9.1 and above users). --target-immediate : Recover ends when a consistent state is reached (end of the base backup) --exclusive : Set target xid to be non inclusive. --tablespace *NAME:LOCATION* : Specify tablespace relocation rule. --remote-ssh-command *SSH_COMMAND* : This options activates remote recovery, by specifying the secure shell command to be launched on a remote host. This is the equivalent of the "ssh_command" server option in the configuration file for remote recovery. Example: 'ssh postgres@db2'. --retry-times *RETRY_TIMES* : Number of retries of data copy during base backup after an error. Overrides value of the parameter `basebackup_retry_times`, if present in the configuration file. --no-retry : Same as `--retry-times 0` --retry-sleep : Number of seconds of wait after a failed copy, before retrying. Overrides value of the parameter `basebackup_retry_sleep`, if present in the configuration file. -j , --jobs : Number of parallel workers to copy files during recovery. Overrides value of the parameter `parallel_jobs`, if present in the configuration file. Works only for servers configured through `rsync`/SSH. --get-wal, --no-get-wal : Enable/Disable usage of `get-wal` for WAL fetching during recovery. Default is based on `recovery_options` setting. --network-compression, --no-network-compression : Enable/Disable network compression during remote recovery. Default is based on `network_compression` configuration setting. get-wal *\[OPTIONS\]* *SERVER_NAME* *WAL_ID* : Retrieve a WAL file from the `xlog` archive of a given server. By default, the requested WAL file, if found, is returned as uncompressed content to `STDOUT`. The following options allow users to change this behaviour: -o *OUTPUT_DIRECTORY* : destination directory where the `get-wal` will deposit the requested WAL -z : output will be compressed using gzip -j : output will be compressed using bzip2 -p *SIZE* : peek from the WAL archive up to *SIZE* WAL files, starting from the requested one. 'SIZE' must be an integer >= 1. When invoked with this option, get-wal returns a list of zero to 'SIZE' WAL segment names, one per row. switch-wal *SERVER_NAME* : Execute pg_switch_wal() on the target server (from PostgreSQL 10), or pg_switch_xlog (for PostgreSQL 8.3 to 9.6). --force : Forces the switch by executing CHECKPOINT before pg_switch_xlog(). *IMPORTANT:* executing a CHECKPOINT might increase I/O load on a PostgreSQL server. Use this option with care. --archive : Wait for one xlog file to be archived. If after a defined amount of time (default: 30 seconds) no xlog file is archived, Barman will teminate with failure exit code. --archive-timeout *TIMEOUT* : Specifies the amount of time in seconds (default: 30 seconds) the archiver will wait for a new xlog file to be archived before timing out. switch-xlog *SERVER_NAME* : Alias for switch-wal (kept for back-compatibility) receive-wal *SERVER_NAME* : Start the stream of transaction logs for a server. The process relies on `pg_receivexlog` to receive WAL files from the PostgreSQL servers through the streaming protocol. --stop : stop the receive-wal process for the server --reset : reset the status of receive-wal, restarting the streaming from the current WAL file of the server --create-slot : create the physical replication slot configured with the `slot_name` configuration parameter --drop-slot : drop the physical replication slot configured with the `slot_name` configuration parameter delete *SERVER_NAME* *BACKUP_ID* : Delete the specified backup. [Backup ID shortcuts](#shortcuts) section below for available shortcuts. replication-status *\[OPTIONS\]* *SERVER_NAME* : Shows live information and status of any streaming client attached to the given server (or servers). Default behaviour can be changed through the following options: --minimal : machine readable output (default: False) --target *TARGET_TYPE* : Possible values for TARGET_TYPE are: - *hot-standby*: lists only hot standby servers - *wal-streamer*: lists only WAL streaming clients, such as pg_receivexlog - *all*: any streaming client (default) # BACKUP ID SHORTCUTS {#shortcuts} Rather than using the timestamp backup ID, you can use any of the following shortcuts/aliases to identity a backup for a given server: first : Oldest available backup for that server, in chronological order. last : Latest available backup for that server, in chronological order. latest : same ast *last*. oldest : same ast *first*. # EXIT STATUS 0 : Success Not zero : Failure # SEE ALSO `barman` (5). # BUGS Barman has been extensively tested, and is currently being used in several production environments. However, we cannot exclude the presence of bugs. Any bug can be reported via the Sourceforge bug tracker. Along the bug submission, users can provide developers with diagnostics information obtained through the `barman diagnose` command. # AUTHORS In alphabetical order: * Gabriele Bartolini (architect) * Jonathan Battiato (QA/testing) * Giulio Calacoci (developer) * Francesco Canovai (QA/testing) * Leonardo Cecchi (developer) * Gianni Ciolli (QA/testing) * Britt Cole (documentation) * Marco Nenciarini (project leader) * Rubens Souza (QA/testing) Past contributors: * Carlo Ascani * Stefano Bianucci * Giuseppe Broccolo # RESOURCES * Homepage: * Documentation: * Professional support: # COPYING Barman is the property of 2ndQuadrant Limited and its code is distributed under GNU General Public License v3. Copyright (C) 2011-2017 2ndQuadrant Limited - . barman-2.3/doc/barman.50000644000076500000240000004517113153300354015453 0ustar mnenciastaff00000000000000.\" Automatically generated by Pandoc 1.19.2.1 .\" .TH "BARMAN" "5" "September 05, 2017" "Barman User manuals" "Version 2.3" .hy .SH NAME .PP barman \- Backup and Recovery Manager for PostgreSQL .SH DESCRIPTION .PP Barman is an administration tool for disaster recovery of PostgreSQL servers written in Python and maintained by 2ndQuadrant. Barman can perform remote backups of multiple servers in business critical environments and helps DBAs during the recovery phase. .SH CONFIGURATION FILE LOCATIONS .PP The system\-level Barman configuration file is located at .IP .nf \f[C] /etc/barman.conf \f[] .fi .PP or .IP .nf \f[C] /etc/barman/barman.conf \f[] .fi .PP and is overridden on a per\-user level by .IP .nf \f[C] $HOME/.barman.conf \f[] .fi .SH CONFIGURATION FILE SYNTAX .PP The Barman configuration file is a plain \f[C]INI\f[] file. There is a general section called \f[C][barman]\f[] and a section \f[C][servername]\f[] for each server you want to backup. Rows starting with \f[C];\f[] are comments. .SH CONFIGURATION FILE DIRECTORY .PP Barman supports the inclusion of multiple configuration files, through the \f[C]configuration_files_directory\f[] option. Included files must contain only server specifications, not global configurations. If the value of \f[C]configuration_files_directory\f[] is a directory, Barman reads all files with \f[C]\&.conf\f[] extension that exist in that folder. For example, if you set it to \f[C]/etc/barman.d\f[], you can specify your PostgreSQL servers placing each section in a separate \f[C]\&.conf\f[] file inside the \f[C]/etc/barman.d\f[] folder. .SH OPTIONS .TP .B active When set to \f[C]true\f[] (default), the server is in full operational state. When set to \f[C]false\f[], the server can be used for diagnostics, but any operational command such as backup execution or WAL archiving is temporarily disabled. Setting \f[C]active=false\f[] is a good practice when adding a new node to Barman. Server. .RS .RE .TP .B archiver This option allows you to activate log file shipping through PostgreSQL\[aq]s \f[C]archive_command\f[] for a server. If set to \f[C]true\f[] (default), Barman expects that continuous archiving for a server is in place and will activate checks as well as management (including compression) of WAL files that Postgres deposits in the \f[I]incoming\f[] directory. Setting it to \f[C]false\f[], will disable standard continuous archiving for a server. Global/Server. .RS .RE .TP .B archiver_batch_size This option allows you to activate batch processing of WAL files for the \f[C]archiver\f[] process, by setting it to a value > 0. Otherwise, the traditional unlimited processing of the WAL queue is enabled. When batch processing is activated, the \f[C]archive\-wal\f[] process would limit itself to maximum \f[C]archiver_batch_size\f[] WAL segments per single run. Integer. Global/Server. .RS .RE .TP .B backup_directory Directory where backup data for a server will be placed. Server. .RS .RE .TP .B backup_method Configure the method barman used for backup execution. If set to \f[C]rsync\f[] (default), barman will execute backup using the \f[C]rsync\f[] command. If set to \f[C]postgres\f[] barman will use the \f[C]pg_basebackup\f[] command to execute the backup. Global/Server. .RS .RE .TP .B backup_options This option allows you to control the way Barman interacts with PostgreSQL for backups. It is a comma\-separated list of values that accepts the following options: .RS .IP \[bu] 2 \f[C]exclusive_backup\f[] (default when \f[C]backup_method\ =\ rsync\f[]): \f[C]barman\ backup\f[] executes backup operations using the standard exclusive backup approach (technically through \f[C]pg_start_backup\f[] and \f[C]pg_stop_backup\f[]) .IP \[bu] 2 \f[C]concurrent_backup\f[] (default when \f[C]backup_method\ =\ postgres\f[]): if using PostgreSQL 9.2, 9.3, 9.4, and 9.5, Barman requires the \f[C]pgespresso\f[] module to be installed on the PostgreSQL server and can be used to perform a backup from a standby server. Starting from PostgreSQL 9.6, Barman uses the new PostgreSQL API to perform backups from a standby server. .IP \[bu] 2 \f[C]external_configuration\f[]: if present, any warning regarding external configuration files is suppressed during the execution of a backup. .PP Note that \f[C]exclusive_backup\f[] and \f[C]concurrent_backup\f[] are mutually exclusive. Global/Server. .RE .TP .B bandwidth_limit This option allows you to specify a maximum transfer rate in kilobytes per second. A value of zero specifies no limit (default). Global/Server. .RS .RE .TP .B barman_home Main data directory for Barman. Global. .RS .RE .TP .B barman_lock_directory Directory for locks. Default: \f[C]%(barman_home)s\f[]. Global. .RS .RE .TP .B basebackups_directory Directory where base backups will be placed. Server. .RS .RE .TP .B basebackup_retry_sleep Number of seconds of wait after a failed copy, before retrying Used during both backup and recovery operations. Positive integer, default 30. Global/Server. .RS .RE .TP .B basebackup_retry_times Number of retries of base backup copy, after an error. Used during both backup and recovery operations. Positive integer, default 0. Global/Server. .RS .RE .TP .B check_timeout Maximum execution time, in seconds per server, for a barman check command. Set to 0 to disable the timeout. Positive integer, default 30. Global/Server. .RS .RE .TP .B compression Standard compression algorithm applied to WAL files. Possible values are: \f[C]gzip\f[] (requires \f[C]gzip\f[] to be installed on the system), \f[C]bzip2\f[] (requires \f[C]bzip2\f[]), \f[C]pigz\f[] (requires \f[C]pigz\f[]), \f[C]pygzip\f[] (Python\[aq]s internal gzip compressor) and \f[C]pybzip2\f[] (Python\[aq]s internal bzip2 compressor). Global/Server. .RS .RE .TP .B conninfo Connection string used by Barman to connect to the Postgres server. This is a libpq connection string, consult the PostgreSQL manual (https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING) for more information. Commonly used keys are: host, hostaddr, port, dbname, user, password. Server. .RS .RE .TP .B custom_compression_filter Customised compression algorithm applied to WAL files. Global/Server. .RS .RE .TP .B custom_decompression_filter Customised decompression algorithm applied to compressed WAL files; this must match the compression algorithm. Global/Server. .RS .RE .TP .B description A human readable description of a server. Server. .RS .RE .TP .B errors_directory Directory that contains WAL files that contain an error; usually this is related to a conflict with an existing WAL file (e.g. a WAL file that has been archived after a streamed one). .RS .RE .TP .B immediate_checkpoint This option allows you to control the way PostgreSQL handles checkpoint at the start of the backup. If set to \f[C]false\f[] (default), the I/O workload for the checkpoint will be limited, according to the \f[C]checkpoint_completion_target\f[] setting on the PostgreSQL server. If set to \f[C]true\f[], an immediate checkpoint will be requested, meaning that PostgreSQL will complete the checkpoint as soon as possible. Global/Server. .RS .RE .TP .B incoming_wals_directory Directory where incoming WAL files are archived into. Requires \f[C]archiver\f[] to be enabled. Server. .RS .RE .TP .B last_backup_maximum_age This option identifies a time frame that must contain the latest backup. If the latest backup is older than the time frame, barman check command will report an error to the user. If empty (default), latest backup is always considered valid. Syntax for this option is: "i (DAYS | WEEKS | MONTHS)" where i is a integer greater than zero, representing the number of days | weeks | months of the time frame. Global/Server. .RS .RE .TP .B log_file Location of Barman\[aq]s log file. Global. .RS .RE .TP .B log_level Level of logging (DEBUG, INFO, WARNING, ERROR, CRITICAL). Global. .RS .RE .TP .B max_incoming_wals_queue Maximum number of WAL files in the incoming queue (in both streaming and archiving pools) that are allowed before barman check returns an error (that does not block backups). Global/Server. Default: None (disabled). .RS .RE .TP .B minimum_redundancy Minimum number of backups to be retained. Default 0. Global/Server. .RS .RE .TP .B network_compression This option allows you to enable data compression for network transfers. If set to \f[C]false\f[] (default), no compression is used. If set to \f[C]true\f[], compression is enabled, reducing network usage. Global/Server. .RS .RE .TP .B parallel_jobs This option controls how many parallel workers will copy files during a backup or recovery command. Default 1. Global/Server. For backup purposes, it works only when \f[C]backup_method\f[] is \f[C]rsync\f[]. .RS .RE .TP .B path_prefix One or more absolute paths, separated by colon, where Barman looks for executable files. The paths specified in \f[C]path_prefix\f[] are tried before the ones specified in \f[C]PATH\f[] environment variable. Global/server. .RS .RE .TP .B post_archive_retry_script Hook script launched after a WAL file is archived by maintenance. Being this a \f[I]retry\f[] hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. In a post archive scenario, ABORT_STOP has currently the same effects as ABORT_CONTINUE. Global/Server. .RS .RE .TP .B post_archive_script Hook script launched after a WAL file is archived by maintenance, after \[aq]post_archive_retry_script\[aq]. Global/Server. .RS .RE .TP .B post_backup_retry_script Hook script launched after a base backup. Being this a \f[I]retry\f[] hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. In a post backup scenario, ABORT_STOP has currently the same effects as ABORT_CONTINUE. Global/Server. .RS .RE .TP .B post_backup_script Hook script launched after a base backup, after \[aq]post_backup_retry_script\[aq]. Global/Server. .RS .RE .TP .B pre_archive_retry_script Hook script launched before a WAL file is archived by maintenance, after \[aq]pre_archive_script\[aq]. Being this a \f[I]retry\f[] hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. Returning ABORT_STOP will propagate the failure at a higher level and interrupt the WAL archiving operation. Global/Server. .RS .RE .TP .B pre_archive_script Hook script launched before a WAL file is archived by maintenance. Global/Server. .RS .RE .TP .B pre_backup_retry_script Hook script launched before a base backup, after \[aq]pre_backup_script\[aq]. Being this a \f[I]retry\f[] hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. Returning ABORT_STOP will propagate the failure at a higher level and interrupt the backup operation. Global/Server. .RS .RE .TP .B pre_backup_script Hook script launched before a base backup. Global/Server. .RS .RE .TP .B recovery_options Options for recovery operations. Currently only supports \f[C]get\-wal\f[]. \f[C]get\-wal\f[] activates generation of a basic \f[C]restore_command\f[] in the resulting \f[C]recovery.conf\f[] file that uses the \f[C]barman\ get\-wal\f[] command to fetch WAL files directly from Barman\[aq]s archive of WALs. Comma separated list of values, default empty. Global/Server. .RS .RE .TP .B retention_policy Policy for retention of periodic backups and archive logs. If left empty, retention policies are not enforced. For redundancy based retention policy use "REDUNDANCY i" (where i is an integer > 0 and defines the number of backups to retain). For recovery window retention policy use "RECOVERY WINDOW OF i DAYS" or "RECOVERY WINDOW OF i WEEKS" or "RECOVERY WINDOW OF i MONTHS" where i is a positive integer representing, specifically, the number of days, weeks or months to retain your backups. For more detailed information, refer to the official documentation. Default value is empty. Global/Server. .RS .RE .TP .B retention_policy_mode Currently only "auto" is implemented. Global/Server. .RS .RE .TP .B reuse_backup This option controls incremental backup support. Global/Server. Possible values are: * \f[C]off\f[]: disabled (default); * \f[C]copy\f[]: reuse the last available backup for a server and create a copy of the unchanged files (reduce backup time); * \f[C]link\f[]: reuse the last available backup for a server and create a hard link of the unchanged files (reduce backup time and space). Requires operating system and file system support for hard links. .RS .RE .TP .B slot_name Physical replication slot to be used by the \f[C]receive\-wal\f[] command when \f[C]streaming_archiver\f[] is set to \f[C]on\f[]. Requires PostgreSQL >= 9.4. Global/Server. Default: None (disabled). .RS .RE .TP .B streaming_archiver This option allows you to use the PostgreSQL\[aq]s streaming protocol to receive transaction logs from a server. If set to \f[C]on\f[], Barman expects to find \f[C]pg_receivexlog\f[] in the PATH (see \f[C]path_prefix\f[] option) and that streaming connection for the server is working. This activates connection checks as well as management (including compression) of WAL files. If set to \f[C]off\f[] (default) barman will rely only on continuous archiving for a server WAL archive operations, eventually terminating any running \f[C]pg_receivexlog\f[] for the server. Global/Server. .RS .RE .TP .B streaming_archiver_batch_size This option allows you to activate batch processing of WAL files for the \f[C]streaming_archiver\f[] process, by setting it to a value > 0. Otherwise, the traditional unlimited processing of the WAL queue is enabled. When batch processing is activated, the \f[C]archive\-wal\f[] process would limit itself to maximum \f[C]streaming_archiver_batch_size\f[] WAL segments per single run. Integer. Global/Server. .RS .RE .TP .B streaming_archiver_name Identifier to be used as \f[C]application_name\f[] by the \f[C]receive\-wal\f[] command. Only available with \f[C]pg_receivexlog\f[] >= 9.3. By default it is set to \f[C]barman_receive_wal\f[]. Global/Server. .RS .RE .TP .B streaming_backup_name Identifier to be used as \f[C]application_name\f[] by the \f[C]pg_basebackup\f[] command. Only available with \f[C]pg_basebackup\f[] >= 9.3. By default it is set to \f[C]barman_streaming_backup\f[]. Global/Server. .RS .RE .TP .B streaming_conninfo Connection string used by Barman to connect to the Postgres server via streaming replication protocol. By default it is set to \f[C]conninfo\f[]. Server. .RS .RE .TP .B streaming_wals_directory Directory where WAL files are streamed from the PostgreSQL server to Barman. Requires \f[C]streaming_archiver\f[] to be enabled. Server. .RS .RE .TP .B ssh_command Command used by Barman to login to the Postgres server via ssh. Server. .RS .RE .TP .B tablespace_bandwidth_limit This option allows you to specify a maximum transfer rate in kilobytes per second, by specifying a comma separated list of tablespaces (pairs TBNAME:BWLIMIT). A value of zero specifies no limit (default). Global/Server. .RS .RE .TP .B wal_retention_policy Policy for retention of archive logs (WAL files). Currently only "MAIN" is available. Global/Server. .RS .RE .TP .B wals_directory Directory which contains WAL files. Server. .RS .RE .SH HOOK SCRIPTS .PP The script definition is passed to a shell and can return any exit code. .PP The shell environment will contain the following variables: .TP .B \f[C]BARMAN_CONFIGURATION\f[] configuration file used by barman .RS .RE .TP .B \f[C]BARMAN_ERROR\f[] error message, if any (only for the \[aq]post\[aq] phase) .RS .RE .TP .B \f[C]BARMAN_PHASE\f[] \[aq]pre\[aq] or \[aq]post\[aq] .RS .RE .TP .B \f[C]BARMAN_RETRY\f[] \f[C]1\f[] if it is a \f[I]retry script\f[] (from 1.5.0), \f[C]0\f[] if not .RS .RE .TP .B \f[C]BARMAN_SERVER\f[] name of the server .RS .RE .PP Backup scripts specific variables: .TP .B \f[C]BARMAN_BACKUP_DIR\f[] backup destination directory .RS .RE .TP .B \f[C]BARMAN_BACKUP_ID\f[] ID of the backup .RS .RE .TP .B \f[C]BARMAN_PREVIOUS_ID\f[] ID of the previous backup (if present) .RS .RE .TP .B \f[C]BARMAN_STATUS\f[] status of the backup .RS .RE .TP .B \f[C]BARMAN_VERSION\f[] version of Barman .RS .RE .PP Archive scripts specific variables: .TP .B \f[C]BARMAN_SEGMENT\f[] name of the WAL file .RS .RE .TP .B \f[C]BARMAN_FILE\f[] full path of the WAL file .RS .RE .TP .B \f[C]BARMAN_SIZE\f[] size of the WAL file .RS .RE .TP .B \f[C]BARMAN_TIMESTAMP\f[] WAL file timestamp .RS .RE .TP .B \f[C]BARMAN_COMPRESSION\f[] type of compression used for the WAL file .RS .RE .PP Only in case of retry hook scripts, the exit code of the script is checked by Barman. Output of hook scripts is simply written in the log file. .SH EXAMPLE .PP Here is an example of configuration file: .IP .nf \f[C] [barman] ;\ Main\ directory barman_home\ =\ /var/lib/barman ;\ System\ user barman_user\ =\ barman ;\ Log\ location log_file\ =\ /var/log/barman/barman.log ;\ Default\ compression\ level ;compression\ =\ gzip ;\ Incremental\ backup reuse_backup\ =\ link ;\ \[aq]main\[aq]\ PostgreSQL\ Server\ configuration [main] ;\ Human\ readable\ description description\ =\ \ "Main\ PostgreSQL\ Database" ;\ SSH\ options ssh_command\ =\ ssh\ postgres\@pg ;\ PostgreSQL\ connection\ string conninfo\ =\ host=pg\ user=postgres ;\ PostgreSQL\ streaming\ connection\ string streaming_conninfo\ =\ host=pg\ user=postgres ;\ Minimum\ number\ of\ required\ backups\ (redundancy) minimum_redundancy\ =\ 1 ;\ Retention\ policy\ (based\ on\ redundancy) retention_policy\ =\ REDUNDANCY\ 2 \f[] .fi .SH SEE ALSO .PP \f[C]barman\f[] (1). .SH AUTHORS .PP In alphabetical order: .IP \[bu] 2 Gabriele Bartolini (architect) .IP \[bu] 2 Jonathan Battiato (QA/testing) .IP \[bu] 2 Giulio Calacoci (developer) .IP \[bu] 2 Francesco Canovai (QA/testing) .IP \[bu] 2 Leonardo Cecchi (developer) .IP \[bu] 2 Gianni Ciolli (QA/testing) .IP \[bu] 2 Britt Cole (documentation) .IP \[bu] 2 Marco Nenciarini (project leader) .IP \[bu] 2 Rubens Souza (QA/testing) .PP Past contributors: .IP \[bu] 2 Carlo Ascani .IP \[bu] 2 Stefano Bianucci .IP \[bu] 2 Giuseppe Broccolo .SH RESOURCES .IP \[bu] 2 Homepage: .IP \[bu] 2 Documentation: .IP \[bu] 2 Professional support: .SH COPYING .PP Barman is the property of 2ndQuadrant Limited and its code is distributed under GNU General Public License v3. .PP Copyright (C) 2011\-2017 2ndQuadrant Limited \- https://www.2ndQuadrant.com/. .SH AUTHORS 2ndQuadrant Limited . barman-2.3/doc/barman.5.md0000644000076500000240000004244413153300354016052 0ustar mnenciastaff00000000000000% BARMAN(5) Barman User manuals | Version 2.3 % 2ndQuadrant Limited % September 05, 2017 # NAME barman - Backup and Recovery Manager for PostgreSQL # DESCRIPTION Barman is an administration tool for disaster recovery of PostgreSQL servers written in Python and maintained by 2ndQuadrant. Barman can perform remote backups of multiple servers in business critical environments and helps DBAs during the recovery phase. # CONFIGURATION FILE LOCATIONS The system-level Barman configuration file is located at /etc/barman.conf or /etc/barman/barman.conf and is overridden on a per-user level by $HOME/.barman.conf # CONFIGURATION FILE SYNTAX The Barman configuration file is a plain `INI` file. There is a general section called `[barman]` and a section `[servername]` for each server you want to backup. Rows starting with `;` are comments. # CONFIGURATION FILE DIRECTORY Barman supports the inclusion of multiple configuration files, through the `configuration_files_directory` option. Included files must contain only server specifications, not global configurations. If the value of `configuration_files_directory` is a directory, Barman reads all files with `.conf` extension that exist in that folder. For example, if you set it to `/etc/barman.d`, you can specify your PostgreSQL servers placing each section in a separate `.conf` file inside the `/etc/barman.d` folder. # OPTIONS active : When set to `true` (default), the server is in full operational state. When set to `false`, the server can be used for diagnostics, but any operational command such as backup execution or WAL archiving is temporarily disabled. Setting `active=false` is a good practice when adding a new node to Barman. Server. archiver : This option allows you to activate log file shipping through PostgreSQL's `archive_command` for a server. If set to `true` (default), Barman expects that continuous archiving for a server is in place and will activate checks as well as management (including compression) of WAL files that Postgres deposits in the *incoming* directory. Setting it to `false`, will disable standard continuous archiving for a server. Global/Server. archiver_batch_size : This option allows you to activate batch processing of WAL files for the `archiver` process, by setting it to a value > 0. Otherwise, the traditional unlimited processing of the WAL queue is enabled. When batch processing is activated, the `archive-wal` process would limit itself to maximum `archiver_batch_size` WAL segments per single run. Integer. Global/Server. backup_directory : Directory where backup data for a server will be placed. Server. backup_method : Configure the method barman used for backup execution. If set to `rsync` (default), barman will execute backup using the `rsync` command. If set to `postgres` barman will use the `pg_basebackup` command to execute the backup. Global/Server. backup_options : This option allows you to control the way Barman interacts with PostgreSQL for backups. It is a comma-separated list of values that accepts the following options: * `exclusive_backup` (default when `backup_method = rsync`): `barman backup` executes backup operations using the standard exclusive backup approach (technically through `pg_start_backup` and `pg_stop_backup`) * `concurrent_backup` (default when `backup_method = postgres`): if using PostgreSQL 9.2, 9.3, 9.4, and 9.5, Barman requires the `pgespresso` module to be installed on the PostgreSQL server and can be used to perform a backup from a standby server. Starting from PostgreSQL 9.6, Barman uses the new PostgreSQL API to perform backups from a standby server. * `external_configuration`: if present, any warning regarding external configuration files is suppressed during the execution of a backup. Note that `exclusive_backup` and `concurrent_backup` are mutually exclusive. Global/Server. bandwidth_limit : This option allows you to specify a maximum transfer rate in kilobytes per second. A value of zero specifies no limit (default). Global/Server. barman_home : Main data directory for Barman. Global. barman_lock_directory : Directory for locks. Default: `%(barman_home)s`. Global. basebackups_directory : Directory where base backups will be placed. Server. basebackup_retry_sleep : Number of seconds of wait after a failed copy, before retrying Used during both backup and recovery operations. Positive integer, default 30. Global/Server. basebackup_retry_times : Number of retries of base backup copy, after an error. Used during both backup and recovery operations. Positive integer, default 0. Global/Server. check_timeout : Maximum execution time, in seconds per server, for a barman check command. Set to 0 to disable the timeout. Positive integer, default 30. Global/Server. compression : Standard compression algorithm applied to WAL files. Possible values are: `gzip` (requires `gzip` to be installed on the system), `bzip2` (requires `bzip2`), `pigz` (requires `pigz`), `pygzip` (Python's internal gzip compressor) and `pybzip2` (Python's internal bzip2 compressor). Global/Server. conninfo : Connection string used by Barman to connect to the Postgres server. This is a libpq connection string, consult the [PostgreSQL manual][conninfo] for more information. Commonly used keys are: host, hostaddr, port, dbname, user, password. Server. [conninfo]: https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING custom_compression_filter : Customised compression algorithm applied to WAL files. Global/Server. custom_decompression_filter : Customised decompression algorithm applied to compressed WAL files; this must match the compression algorithm. Global/Server. description : A human readable description of a server. Server. errors_directory : Directory that contains WAL files that contain an error; usually this is related to a conflict with an existing WAL file (e.g. a WAL file that has been archived after a streamed one). immediate_checkpoint : This option allows you to control the way PostgreSQL handles checkpoint at the start of the backup. If set to `false` (default), the I/O workload for the checkpoint will be limited, according to the `checkpoint_completion_target` setting on the PostgreSQL server. If set to `true`, an immediate checkpoint will be requested, meaning that PostgreSQL will complete the checkpoint as soon as possible. Global/Server. incoming_wals_directory : Directory where incoming WAL files are archived into. Requires `archiver` to be enabled. Server. last_backup_maximum_age : This option identifies a time frame that must contain the latest backup. If the latest backup is older than the time frame, barman check command will report an error to the user. If empty (default), latest backup is always considered valid. Syntax for this option is: "i (DAYS | WEEKS | MONTHS)" where i is a integer greater than zero, representing the number of days | weeks | months of the time frame. Global/Server. log_file : Location of Barman's log file. Global. log_level : Level of logging (DEBUG, INFO, WARNING, ERROR, CRITICAL). Global. max_incoming_wals_queue : Maximum number of WAL files in the incoming queue (in both streaming and archiving pools) that are allowed before barman check returns an error (that does not block backups). Global/Server. Default: None (disabled). minimum_redundancy : Minimum number of backups to be retained. Default 0. Global/Server. network_compression : This option allows you to enable data compression for network transfers. If set to `false` (default), no compression is used. If set to `true`, compression is enabled, reducing network usage. Global/Server. parallel_jobs : This option controls how many parallel workers will copy files during a backup or recovery command. Default 1. Global/Server. For backup purposes, it works only when `backup_method` is `rsync`. path_prefix : One or more absolute paths, separated by colon, where Barman looks for executable files. The paths specified in `path_prefix` are tried before the ones specified in `PATH` environment variable. Global/server. post_archive_retry_script : Hook script launched after a WAL file is archived by maintenance. Being this a _retry_ hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. In a post archive scenario, ABORT_STOP has currently the same effects as ABORT_CONTINUE. Global/Server. post_archive_script : Hook script launched after a WAL file is archived by maintenance, after 'post_archive_retry_script'. Global/Server. post_backup_retry_script : Hook script launched after a base backup. Being this a _retry_ hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. In a post backup scenario, ABORT_STOP has currently the same effects as ABORT_CONTINUE. Global/Server. post_backup_script : Hook script launched after a base backup, after 'post_backup_retry_script'. Global/Server. pre_archive_retry_script : Hook script launched before a WAL file is archived by maintenance, after 'pre_archive_script'. Being this a _retry_ hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. Returning ABORT_STOP will propagate the failure at a higher level and interrupt the WAL archiving operation. Global/Server. pre_archive_script : Hook script launched before a WAL file is archived by maintenance. Global/Server. pre_backup_retry_script : Hook script launched before a base backup, after 'pre_backup_script'. Being this a _retry_ hook script, Barman will retry the execution of the script until this either returns a SUCCESS (0), an ABORT_CONTINUE (62) or an ABORT_STOP (63) code. Returning ABORT_STOP will propagate the failure at a higher level and interrupt the backup operation. Global/Server. pre_backup_script : Hook script launched before a base backup. Global/Server. recovery_options : Options for recovery operations. Currently only supports `get-wal`. `get-wal` activates generation of a basic `restore_command` in the resulting `recovery.conf` file that uses the `barman get-wal` command to fetch WAL files directly from Barman's archive of WALs. Comma separated list of values, default empty. Global/Server. retention_policy : Policy for retention of periodic backups and archive logs. If left empty, retention policies are not enforced. For redundancy based retention policy use "REDUNDANCY i" (where i is an integer > 0 and defines the number of backups to retain). For recovery window retention policy use "RECOVERY WINDOW OF i DAYS" or "RECOVERY WINDOW OF i WEEKS" or "RECOVERY WINDOW OF i MONTHS" where i is a positive integer representing, specifically, the number of days, weeks or months to retain your backups. For more detailed information, refer to the official documentation. Default value is empty. Global/Server. retention_policy_mode : Currently only "auto" is implemented. Global/Server. reuse_backup : This option controls incremental backup support. Global/Server. Possible values are: * `off`: disabled (default); * `copy`: reuse the last available backup for a server and create a copy of the unchanged files (reduce backup time); * `link`: reuse the last available backup for a server and create a hard link of the unchanged files (reduce backup time and space). Requires operating system and file system support for hard links. slot_name : Physical replication slot to be used by the `receive-wal` command when `streaming_archiver` is set to `on`. Requires PostgreSQL >= 9.4. Global/Server. Default: None (disabled). streaming_archiver : This option allows you to use the PostgreSQL's streaming protocol to receive transaction logs from a server. If set to `on`, Barman expects to find `pg_receivexlog` in the PATH (see `path_prefix` option) and that streaming connection for the server is working. This activates connection checks as well as management (including compression) of WAL files. If set to `off` (default) barman will rely only on continuous archiving for a server WAL archive operations, eventually terminating any running `pg_receivexlog` for the server. Global/Server. streaming_archiver_batch_size : This option allows you to activate batch processing of WAL files for the `streaming_archiver` process, by setting it to a value > 0. Otherwise, the traditional unlimited processing of the WAL queue is enabled. When batch processing is activated, the `archive-wal` process would limit itself to maximum `streaming_archiver_batch_size` WAL segments per single run. Integer. Global/Server. streaming_archiver_name : Identifier to be used as `application_name` by the `receive-wal` command. Only available with `pg_receivexlog` >= 9.3. By default it is set to `barman_receive_wal`. Global/Server. streaming_backup_name : Identifier to be used as `application_name` by the `pg_basebackup` command. Only available with `pg_basebackup` >= 9.3. By default it is set to `barman_streaming_backup`. Global/Server. streaming_conninfo : Connection string used by Barman to connect to the Postgres server via streaming replication protocol. By default it is set to `conninfo`. Server. streaming_wals_directory : Directory where WAL files are streamed from the PostgreSQL server to Barman. Requires `streaming_archiver` to be enabled. Server. ssh_command : Command used by Barman to login to the Postgres server via ssh. Server. tablespace_bandwidth_limit : This option allows you to specify a maximum transfer rate in kilobytes per second, by specifying a comma separated list of tablespaces (pairs TBNAME:BWLIMIT). A value of zero specifies no limit (default). Global/Server. wal_retention_policy : Policy for retention of archive logs (WAL files). Currently only "MAIN" is available. Global/Server. wals_directory : Directory which contains WAL files. Server. # HOOK SCRIPTS The script definition is passed to a shell and can return any exit code. The shell environment will contain the following variables: `BARMAN_CONFIGURATION` : configuration file used by barman `BARMAN_ERROR` : error message, if any (only for the 'post' phase) `BARMAN_PHASE` : 'pre' or 'post' `BARMAN_RETRY` : `1` if it is a _retry script_ (from 1.5.0), `0` if not `BARMAN_SERVER` : name of the server Backup scripts specific variables: `BARMAN_BACKUP_DIR` : backup destination directory `BARMAN_BACKUP_ID` : ID of the backup `BARMAN_PREVIOUS_ID` : ID of the previous backup (if present) `BARMAN_STATUS` : status of the backup `BARMAN_VERSION` : version of Barman Archive scripts specific variables: `BARMAN_SEGMENT` : name of the WAL file `BARMAN_FILE` : full path of the WAL file `BARMAN_SIZE` : size of the WAL file `BARMAN_TIMESTAMP` : WAL file timestamp `BARMAN_COMPRESSION` : type of compression used for the WAL file Only in case of retry hook scripts, the exit code of the script is checked by Barman. Output of hook scripts is simply written in the log file. # EXAMPLE Here is an example of configuration file: ``` [barman] ; Main directory barman_home = /var/lib/barman ; System user barman_user = barman ; Log location log_file = /var/log/barman/barman.log ; Default compression level ;compression = gzip ; Incremental backup reuse_backup = link ; 'main' PostgreSQL Server configuration [main] ; Human readable description description = "Main PostgreSQL Database" ; SSH options ssh_command = ssh postgres@pg ; PostgreSQL connection string conninfo = host=pg user=postgres ; PostgreSQL streaming connection string streaming_conninfo = host=pg user=postgres ; Minimum number of required backups (redundancy) minimum_redundancy = 1 ; Retention policy (based on redundancy) retention_policy = REDUNDANCY 2 ``` # SEE ALSO `barman` (1). # AUTHORS In alphabetical order: * Gabriele Bartolini (architect) * Jonathan Battiato (QA/testing) * Giulio Calacoci (developer) * Francesco Canovai (QA/testing) * Leonardo Cecchi (developer) * Gianni Ciolli (QA/testing) * Britt Cole (documentation) * Marco Nenciarini (project leader) * Rubens Souza (QA/testing) Past contributors: * Carlo Ascani * Stefano Bianucci * Giuseppe Broccolo # RESOURCES * Homepage: * Documentation: * Professional support: # COPYING Barman is the property of 2ndQuadrant Limited and its code is distributed under GNU General Public License v3. Copyright (C) 2011-2017 2ndQuadrant Limited - https://www.2ndQuadrant.com/. barman-2.3/doc/barman.conf0000644000076500000240000000536113113023325016225 0ustar mnenciastaff00000000000000; Barman, Backup and Recovery Manager for PostgreSQL ; http://www.pgbarman.org/ - http://www.2ndQuadrant.com/ ; ; Main configuration file [barman] ; System user barman_user = barman ; Directory of configuration files. Place your sections in separate files with .conf extension ; For example place the 'main' server section in /etc/barman.d/main.conf configuration_files_directory = /etc/barman.d ; Main directory barman_home = /var/lib/barman ; Locks directory - default: %(barman_home)s ;barman_lock_directory = /var/run/barman ; Log location log_file = /var/log/barman/barman.log ; Log level (see https://docs.python.org/3/library/logging.html#levels) log_level = INFO ; Default compression level: possible values are None (default), bzip2, gzip, pigz, pygzip or pybzip2 ;compression = gzip ; Pre/post backup hook scripts ;pre_backup_script = env | grep ^BARMAN ;pre_backup_retry_script = env | grep ^BARMAN ;post_backup_retry_script = env | grep ^BARMAN ;post_backup_script = env | grep ^BARMAN ; Pre/post archive hook scripts ;pre_archive_script = env | grep ^BARMAN ;pre_archive_retry_script = env | grep ^BARMAN ;post_archive_retry_script = env | grep ^BARMAN ;post_archive_script = env | grep ^BARMAN ; Global retention policy (REDUNDANCY or RECOVERY WINDOW) - default empty ;retention_policy = ; Global bandwidth limit in KBPS - default 0 (meaning no limit) ;bandwidth_limit = 4000 ; Number of parallel jobs for backup and recovery via rsync (default 1) ;parallel_jobs = 1 ; Immediate checkpoint for backup command - default false ;immediate_checkpoint = false ; Enable network compression for data transfers - default false ;network_compression = false ; Number of retries of data copy during base backup after an error - default 0 ;basebackup_retry_times = 0 ; Number of seconds of wait after a failed copy, before retrying - default 30 ;basebackup_retry_sleep = 30 ; Maximum execution time, in seconds, per server ; for a barman check command - default 30 ;check_timeout = 30 ; Time frame that must contain the latest backup date. ; If the latest backup is older than the time frame, barman check ; command will report an error to the user. ; If empty, the latest backup is always considered valid. ; Syntax for this option is: "i (DAYS | WEEKS | MONTHS)" where i is an ; integer > 0 which identifies the number of days | weeks | months of ; validity of the latest backup for this check. Also known as 'smelly backup'. ;last_backup_maximum_age = ; Minimum number of required backups (redundancy) ;minimum_redundancy = 1 ; Examples of retention policies ; Retention policy (disabled) ;retention_policy = ; Retention policy (based on redundancy) ;retention_policy = REDUNDANCY 2 ; Retention policy (based on recovery window) ;retention_policy = RECOVERY WINDOW OF 4 WEEKS barman-2.3/doc/barman.d/0000755000076500000240000000000013153300776015607 5ustar mnenciastaff00000000000000barman-2.3/doc/barman.d/ssh-server.conf-template0000644000076500000240000000273413113023325022362 0ustar mnenciastaff00000000000000; Barman, Backup and Recovery Manager for PostgreSQL ; http://www.pgbarman.org/ - http://www.2ndQuadrant.com/ ; ; Template configuration file for a server using ; SSH connections and rsync for copy. ; [ssh] ; Human readable description description = "Example of PostgreSQL Database (via SSH)" ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; SSH options (mandatory) ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ssh_command = ssh postgres@pg ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; PostgreSQL connection string (mandatory) ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; conninfo = host=pg user=barman dbname=postgres ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Backup settings (via rsync over SSH) ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; backup_method = rsync ; Incremental backup support: possible values are None (default), link or copy ;reuse_backup = link ; Identify the standard behavior for backup operations: possible values are ; exclusive_backup (default), concurrent_backup ;backup_options = exclusive_backup ; Number of parallel workers to perform file copy during backup and recover ;parallel_jobs = 1 ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Continuous WAL archiving (via 'archive_command') ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; archiver = on ;archiver_batch_size = 50 ; PATH setting for this server ;path_prefix = "/usr/pgsql-9.6/bin" barman-2.3/doc/barman.d/streaming-server.conf-template0000644000076500000240000000270113066203575023566 0ustar mnenciastaff00000000000000; Barman, Backup and Recovery Manager for PostgreSQL ; http://www.pgbarman.org/ - http://www.2ndQuadrant.com/ ; ; Template configuration file for a server using ; only streaming replication protocol ; [streaming] ; Human readable description description = "Example of PostgreSQL Database (Streaming-Only)" ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; PostgreSQL connection string (mandatory) ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; conninfo = host=pg user=barman dbname=postgres ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; PostgreSQL streaming connection string ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; To be used by pg_basebackup for backup and pg_receivexlog for WAL streaming ; NOTE: streaming_barman is a regular user with REPLICATION privilege streaming_conninfo = host=pg user=streaming_barman ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; Backup settings (via pg_basebackup) ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; backup_method = postgres ;streaming_backup_name = barman_streaming_backup ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ; WAL streaming settings (via pg_receivexlog) ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; streaming_archiver = on slot_name = barman ;streaming_archiver_name = barman_receive_wal ;streaming_archiver_batch_size = 50 ; PATH setting for this server ;path_prefix = "/usr/pgsql-9.6/bin" barman-2.3/doc/Makefile0000644000076500000240000000067013066203575015572 0ustar mnenciastaff00000000000000MANPAGES=barman.1 barman.5 DOCS= SUBDIRS=manual .PHONY: all clean help subdirs $(SUBDIRS) all: $(MANPAGES) $(DOCS) $(SUBDIRS) barman.1: barman.1.md pandoc -s -t man -o $@ $< barman.5: barman.5.md pandoc -s -t man -o $@ $< clean: rm -f $(MANPAGES) $(DOCS) for dir in $(SUBDIRS); do \ $(MAKE) -C $$dir clean; \ done help: @echo "Usage:" @echo " $$ make" subdirs: $(SUBDIRS) $(SUBDIRS): $(MAKE) -C $@ barman-2.3/doc/manual/0000755000076500000240000000000013153300776015402 5ustar mnenciastaff00000000000000barman-2.3/doc/manual/00-head.en.md0000644000076500000240000000142213153300354017432 0ustar mnenciastaff00000000000000% Barman Manual % 2ndQuadrant Limited % September 05, 2017 (v2.3) **Barman** (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. [Barman] [11] is distributed under GNU GPL 3 and maintained by [2ndQuadrant] [13], a platinum sponsor of the [PostgreSQL project] [31]. > **IMPORTANT:** \newline > This manual assumes that you are familiar with theoretical disaster > recovery concepts, and that you have a grasp of PostgreSQL fundamentals in > terms of physical backup and disaster recovery. See section _"Before you start"_ below for details. barman-2.3/doc/manual/01-intro.en.md0000644000076500000240000000766113066203575017712 0ustar mnenciastaff00000000000000\newpage # Introduction In a perfect world, there would be no need for a backup. However, it is important, especially in business environments, to be prepared for when the _"unexpected"_ happens. In a database scenario, the unexpected could take any of the following forms: - data corruption - system failure (including hardware failure) - human error - natural disaster In such cases, any ICT manager or DBA should be able to fix the incident and recover the database in the shortest time possible. We normally refer to this discipline as **disaster recovery**, and more broadly *business continuity*. Within business continuity, it is important to familiarise with two fundamental metrics, as defined by Wikipedia: - [**Recovery Point Objective (RPO)**] [rpo]: _"maximum targeted period in which data might be lost from an IT service due to a major incident"_ - [**Recovery Time Objective (RTO)**] [rto]: _"the targeted duration of time and a service level within which a business process must be restored after a disaster (or disruption) in order to avoid unacceptable consequences associated with a break in business continuity"_ In a few words, RPO represents the maximum amount of data you can afford to lose, while RTO represents the maximum down-time you can afford for your service. Understandably, we all want **RPO=0** (*"zero data loss"*) and **RTO=0** (*zero down-time*, utopia) - even if it is our grandmothers's recipe website. In reality, a careful cost analysis phase allows you to determine your business continuity requirements. Fortunately, with an open source stack composed of **Barman** and **PostgreSQL**, you can achieve RPO=0 thanks to synchronous streaming replication. RTO is more the focus of a *High Availability* solution, like [**repmgr**] [repmgr]. Therefore, by integrating Barman and repmgr, you can dramatically reduce RTO to nearly zero. Based on our experience at 2ndQuadrant, we can confirm that PostgreSQL open source clusters with Barman and repmgr can easily achieve more than 99.99% uptime over a year, if properly configured and monitored. In any case, it is important for us to emphasise more on cultural aspects related to disaster recovery, rather than the actual tools. Tools without human beings are useless. Our mission with Barman is to promote a culture of disaster recovery that: - focuses on backup procedures - focuses even more on recovery procedures - relies on education and training on strong theoretical and practical concepts of PostgreSQL's crash recovery, backup, Point-In-Time-Recovery, and replication for your team members - promotes testing your backups (only a backup that is tested can be considered to be valid), either manually or automatically (be creative with Barman's hook scripts!) - fosters regular practice of recovery procedures, by all members of your devops team (yes, developers too, not just system administrators and DBAs) - solicites to regularly scheduled drills and disaster recovery simulations with the team every 3-6 months - relies on continuous monitoring of PostgreSQL and Barman, and that is able to promptly identify any anomalies Moreover, do everything you can to prepare yourself and your team for when the disaster happens (yes, *when*), because when it happens: - It is going to be a Friday evening, most likely right when you are about to leave the office. - It is going to be when you are on holiday (right in the middle of your cruise around the world) and somebody else has to deal with it. - It is certainly going to be stressful. - You will regret not being sure that the last available backup is valid. - Unless you know how long it approximately takes to recover, every second will seems like forever. Be prepared, don't be scared. In 2011, with these goals in mind, 2ndQuadrant started the development of Barman, now one of the most used backup tools for PostgreSQL. Barman is an acronym for "Backup and Recovery Manager". Currently, Barman works only on Linux and Unix operating systems. barman-2.3/doc/manual/02-before_you_start.en.md0000644000076500000240000000205313066203575022121 0ustar mnenciastaff00000000000000\newpage # Before you start Before you start using Barman, it is fundamental that you get familiar with PostgreSQL and the concepts around physical backups, Point-In-Time-Recovery and replication, such as base backups, WAL archiving, etc. Below you can find a non exhaustive list of resources that we recommend for you to read: - _PostgreSQL documentation_: - [SQL Dump] [sqldump] [^pgdump] - [File System Level Backup] [physicalbackup] - [Continuous Archiving and Point-in-Time Recovery (PITR)] [pitr] - [Recovery Configuration] [recoveryconfig] - [Reliability and the Write-Ahead Log] [wal] - _Book_: [PostgreSQL 9 Administration Cookbook - 2nd edition] [adminbook] [^pgdump]: It is important that you know the difference between logical and physical backup, therefore between `pg_dump` and a tool like Barman. Professional training on these topics is another effective way of learning these concepts. At any time of the year you can find many courses available all over the world, delivered by PostgreSQL companies such as 2ndQuadrant. barman-2.3/doc/manual/10-design.en.md0000644000076500000240000002323213113023325020002 0ustar mnenciastaff00000000000000\newpage # Design and architecture ## Where to install Barman One of the foundations of Barman is the ability to operate remotely from the database server, via the network. Theoretically, you could have your Barman server located in a data centre in another part of the world, thousands of miles away from your PostgreSQL server. Realistically, you do not want your Barman server to be too far from your PostgreSQL server, so that both backup and recovery times are kept under control. Even though there is no _"one size fits all"_ way to setup Barman, there are a couple of recommendations that we suggest you abide by, in particular: - Install Barman on a dedicated server - Do not share the same storage with your PostgreSQL server - Integrate Barman with your monitoring infrastructure [^nagios] - Test everything before you deploy it to production [^nagios]: Integration with Nagios/Icinga is straightforward thanks to the `barman check --nagios` command, one of the most important features of Barman and a true lifesaver. A reasonable way to start modelling your disaster recovery architecture is to: - design a couple of possibile architectures in respect to PostgreSQL and Barman, such as: 1. same data centre 2. different data centre in the same metropolitan area 3. different data centre - elaborate the pros and the cons of each hypothesis - evaluate the single points of failure (SPOF) of your system, with cost-benefit analysis - make your decision and implement the initial solution Having said this, a very common setup for Barman is to be installed in the same data centre where your PostgreSQL servers are. In this case, the single point of failure is the data centre. Fortunately, the impact of such a SPOF can be alleviated thanks to a feature called _hook scripts_. Indeed, backups of Barman can be exported on different media, such as _tape_ via `tar`, or locations, like an _S3 bucket_ in the Amazon cloud. Remember that no decision is forever. You can start this way and adapt over time to the solution that suits you best. However, try and keep it simple to start with. ## One Barman, many PostgreSQL servers Another relevant feature that was first introduced by Barman is support for multiple servers. Barman can store backup data coming from multiple PostgreSQL instances, even with different versions, in a centralised way. [^recver] [^recver]: The same [requirements for PostgreSQL's PITR][requirements_recovery] apply for recovery, as detailed in the section _"Requirements for recovery"_. As a result, you can model complex disaster recovery architectures, forming a "star schema", where PostgreSQL servers rotate around a central Barman server. Every architecture makes sense in its own way. Choose the one that resonates with you, and most importantly, the one you trust, based on real experimentation and testing. From this point forward, for the sake of simplicity, this guide will assume a basic architecture: - one PostgreSQL instance (with host name `pg`) - one backup server with Barman (with host name `backup`) ## Streaming backup vs rsync/SSH Traditionally, Barman has always operated remotely via SSH, taking advantage of `rsync` for physical backup operations. Version 2.0 introduces native support for PostgreSQL's streaming replication protocol for backup operations, via `pg_basebackup`. [^fmatrix] [^fmatrix]: Check in the "Feature matrix" which PostgreSQL versions support streaming replication backups with Barman. Choosing one of these two methods is a decision you will need to make. On a general basis, starting from Barman 2.0, backup over streaming replication is the recommended setup for PostgreSQL 9.4 or higher. Moreover, if you do not make use of tablespaces, backup over streaming can be used starting from PostgreSQL 9.2. > **IMPORTANT:** \newline > Because Barman transparently makes use of `pg_basebackup`, features such as incremental backup, parallel backup, deduplication, and network compression are currently not available. In this case, bandwidth limitation has some restrictions - compared to the traditional method via `rsync`. Traditional backup via `rsync`/SSH is available for all versions of PostgreSQL starting from 8.3, and it is recommended in all cases where `pg_basebackup` limitations occur (for example, a very large database that can benefit from incremental backup and deduplication). The reason why we recommend streaming backup is that, based on our experience, it is easier to setup than the traditional one. Also, streaming backup allows you to backup a PostgreSQL server on Windows[^windows], and makes life easier when working with Docker. [^windows]: Backup of a PostgreSQL server on Windows is possible, but it is still experimental because it is not yet part of our continuous integration system. See section _"How to setup a Windows based server"_ for details. ## Standard archiving, WAL streaming ... or both PostgreSQL's Point-In-Time-Recovery requires that transactional logs, also known as _xlog_ or WAL files, are stored alongside of base backups. Traditionally, Barman has supported standard WAL file shipping through PostgreSQL's `archive_command` (usually via `rsync`/SSH). With this method, WAL files are archived only when PostgreSQL _switches_ to a new WAL file. To keep it simple, this normally happens every 16MB worth of data changes. Barman 1.6.0 introduces streaming of WAL files for PostgreSQL servers 9.2 or higher, as an additional method for transactional log archiving, through `pg_receivexlog`. WAL streaming is able to reduce the risk of data loss, bringing RPO down to _near zero_ values. Barman 2.0 introduces support for replication slots with PostgreSQL servers 9.4 or above, therefore allowing WAL streaming-only configurations. Moreover, you can now add Barman as a synchronous WAL receiver in your PostgreSQL 9.5 (or higher) cluster, and achieve **zero data loss** (RPO=0). In some cases you have no choice and you are forced to use traditional archiving. In others, you can choose whether to use both or just WAL streaming. Unless you have strong reasons not to do it, we recommend to use both channels, for maximum reliability and robustness. ## Two typical scenarios for backups In order to make life easier for you, below we summarise the two most typical scenarios for a given PostgreSQL server in Barman. Bear in mind that this is a decision that you must make for every single server that you decide to back up with Barman. This means that you can have heterogeneous setups within the same installation. As mentioned before, we will only worry about the PostgreSQL server (`pg`) and the Barman server (`backup`). However, in real life, your architecture will most likely contain other technologies such as repmgr, pgBouncer, Nagios/Icinga, and so on. ### Scenario 1: Backup via streaming protocol If you are using PostgreSQL 9.4 or higher, and your database falls under a general use case scenario, you will likely end up deciding on a streaming backup installation - see figure \ref{scenario1-design} below. ![Streaming-only backup (Scenario 1)\label{scenario1-design}](../images/barman-architecture-scenario1.png){ width=80% } In this scenario, you will need to configure: 1. a standard connection to PostgreSQL, for management, coordination, and monitoring purposes 2. a streaming replication connection that will be used by both `pg_basebackup` (for base backup operations) and `pg_receivexlog` (for WAL streaming) This setup, in Barman's terminology, is known as **streaming-only** setup, as it does not require any SSH connection for backup and archiving operations. This is particularly suitable and extremely practical for Docker environments. However, as mentioned before, you can configure standard archiving as well and implement a more robust architecture - see figure \ref{scenario1b-design} below. ![Streaming backup with WAL archiving (Scenario 1b)\label{scenario1b-design}](../images/barman-architecture-scenario1b.png){ width=80% } This alternate approach requires: - an additional SSH connection that allows the `postgres` user on the PostgreSQL server to connect as `barman` user on the Barman server - the `archive_command` in PostgreSQL be configured to ship WAL files to Barman This architecture is available also to PostgreSQL 9.2/9.3 users that do not use tablespaces. ### Scenario 2: Backup via `rsync`/SSH The _traditional_ setup of `rsync` over SSH is the only available option for: - PostgreSQL servers version 8.3, 8.4, 9.0 or 9.1 - PostgreSQL servers version 9.2 or 9.3 that are using tablespaces - incremental backup, parallel backup and deduplication - network compression during backups - finer control of bandwidth usage, including on a tablespace basis ![Scenario 2 - Backup via rsync/SSH](../images/barman-architecture-scenario2.png){ width=80% } In this scenario, you will need to configure: 1. a standard connection to PostgreSQL for management, coordination, and monitoring purposes 2. an SSH connection for base backup operations to be used by `rsync` that allows the `barman` user on the Barman server to connect as `postgres` user on the PostgreSQL server 3. an SSH connection for WAL archiving to be used by the `archive_command` in PostgreSQL and that allows the `postgres` user on the PostgreSQL server to connect as `barman` user on the Barman server Starting from PostgreSQL 9.2, you can add a streaming replication connection that is used for WAL streaming and significantly reduce RPO. This more robust implementation is depicted in figure \ref{scenario2b-design}. ![Backup via rsync/SSH with WAL streaming (Scenario 2b)\label{scenario2b-design}](../images/barman-architecture-scenario2b.png){ width=80% } barman-2.3/doc/manual/15-system_requirements.en.md0000644000076500000240000000452013066203575022702 0ustar mnenciastaff00000000000000\newpage # System requirements - Linux/Unix - Python 2.6 or 2.7 - Python modules: - argcomplete - argh >= 0.21.2 <= 0.26.2 - argparse (Python 2.6 only) - psycopg2 >= 2.4.2 - python-dateutil <> 2.0 - setuptools - PostgreSQL >= 8.3 - rsync >= 3.0.4 (optional for PostgreSQL >= 9.2) > **IMPORTANT:** > Users of RedHat Enterprise Linux, CentOS and Scientific Linux are > required to install the > [Extra Packages Enterprise Linux (EPEL) repository] [epel]. > **NOTE:** > Python 3 support is experimental. Report any bug through > the ticketing system on Github or the mailing list. ## Requirements for backup The most critical requirement for a Barman server is the amount of disk space available. You are recommended to plan the required disk space based on the size of the cluster, number of WAL files generated per day, frequency of backups, and retention policies. Although the only file systems that we officially support are XFS and Ext4, we are aware of users that deploy Barman on different file systems including ZFS and NFS. ## Requirements for recovery Barman allows you to recover a PostgreSQL instance either locally (where Barman resides) or remotely (on a separate server). Remote recovery is definitely the most common way to restore a PostgreSQL server with Barman. Either way, the same [requirements for PostgreSQL's Log shipping and Point-In-Time-Recovery apply] [requirements_recovery]: - identical hardware architecture - identical major version of PostgreSQL In general, it is **highly recommended** to create recovery environments that are as similar as possible, if not identical, to the original server, because they are easier to maintain. For example, we suggest that you use the same operating system, the same PostgreSQL version, the same disk layouts, and so on. Additionally, dedicated recovery environments for each PostgreSQL server, even on demand, allows you to nurture the disaster recovery culture in your team. You can be prepared for when something unexpected happens by practising recovery operations and becoming familiar with them. Based on our experience, designated recovery environments reduce the impact of stress in real failure situations, and therefore increase the effectiveness of recovery operations. Finally, it is important that time is synchronised between the servers, using NTP for example. barman-2.3/doc/manual/16-installation.en.md0000644000076500000240000000626713066203575021267 0ustar mnenciastaff00000000000000\newpage # Installation > **IMPORTANT:** > The recommended way to install Barman is by using the available > packages for your GNU/Linux distribution. ## Installation on RedHat/CentOS using RPM packages Barman can be installed on RHEL7, RHEL6 and RHEL5 Linux systems using RPM packages. It is required to install the Extra Packages Enterprise Linux (EPEL) repository beforehand. RPM packages for Barman are available via Yum through the [PostgreSQL Global Development Group RPM repository] [yumpgdg]. You need to follow the instructions for your distribution (for example RedHat, CentOS, or Fedora) and architecture as detailed at [yum.postgresql.org] [yumpgdg]. Then, as `root` simply type: ``` bash yum install barman ``` 2ndQuadrant also maintains RPM packages for Barman and distributes them through [Sourceforge.net] [3]. ## Installation on Debian/Ubuntu using packages Barman can be installed on Debian and Ubuntu Linux systems using packages. It is directly available in the official repository for Debian and Ubuntu, however, these repositories might not contain the latest available version. If you want to have the latest version of Barman, the recommended method is to install it through the [PostgreSQL Community APT repository] [aptpgdg]. Instructions can be found in the [APT section of the PostgreSQL Wiki] [aptpgdgwiki]. > **NOTE:** > Thanks to the direct involvement of Barman developers in the > PostgreSQL Community APT repository project, you will always have access > to the most updated versions of Barman. Installing Barman is as easy. As `root` user simply type: ``` bash apt-get install barman ``` ## Installation from sources > **WARNING:** > Manual installation of Barman from sources should only be performed > by expert GNU/Linux users. Installing Barman this way requires > system administration activities such as dependencies management, > `barman` user creation, configuration of the `barman.conf` file, > cron setup for the `barman cron` command, log management, and so on. Create a system user called `barman` on the `backup` server. As `barman` user, download the sources and uncompress them. For a system-wide installation, type: ``` bash barman@backup$ ./setup.py build # run this command with root privileges or through sudo barman@backup# ./setup.py install ``` For a local installation, type: ``` bash barman@backup$ ./setup.py install --user ``` The `barman` application will be installed in your user directory ([make sure that your `PATH` environment variable is set properly] [setup_user]). [Barman is also available on the Python Package Index (PyPI)] [pypi] and can be installed through `pip`. ## Upgrading from Barman 1.X Version 2.0 requires that users explicitly configure their archiving strategy. Before, the file based archiver, controlled by `archiver`, was enabled by default. When you upgrade your Barman installation to 2.0, make sure you add the following line either globally or for any server that requires it: ``` ini archiver = on ``` Additionally, for a few releases, Barman will transparently set `archiver = on` with any server that has not explicitly set an archiving strategy and emit a warning. Besides that, version 2.0 is fully compatible with older ones. barman-2.3/doc/manual/17-configuration.en.md0000644000076500000240000000675513113023325021422 0ustar mnenciastaff00000000000000\newpage # Configuration There are two types of configuration files in Barman: - **global/general configuration** - **server configuration** The main configuration file (set to `/etc/barman.conf` by default) contains general options such as main directory, system user, log file, and so on. Server configuration files, one for each server to be backed up by Barman, are located in the `/etc/barman.d` directory and must have a `.conf` suffix. > **IMPORTANT**: For historical reasons, you can still have one single > configuration file containing both global and server options. However, > for maintenance reasons, this approach is deprecated. Configuration files in Barman follow the _INI_ format. Configuration files accept distinct types of parameters: - string - enum - integer - boolean, `on/true/1` are accepted as well are `off/false/0`. None of them requires to be quoted. > *NOTE*: some `enum` allows `off` but not `false`. ## Options scope Every configuration option has a _scope_: - global - server - global/server: server options that can be generally set at global level Global options are allowed in the _general section_, which is identified in the INI file by the `[barman]` label: ``` ini [barman] ; ... global and global/server options go here ``` Server options can only be specified in a _server section_, which is identified by a line in the configuration file, in square brackets (`[` and `]`). The server section represents the ID of that server in Barman. The following example specifies a section for the server named `pg`: ``` ini [pg] ; Configuration options for the ; server named 'pg' go here ``` There are two reserved words that cannot be used as server names in Barman: - `barman`: identifier of the global section - `all`: a handy shortcut that allows you to execute some commands on every server managed by Barman in sequence Barman implements the **convention over configuration** design paradigm, which attempts to reduce the number of options that you are required to configure without losing flexibility. Therefore, some server options can be defined at global level and overridden at server level, allowing users to specify a generic behavior and refine it for one or more servers. These options have a global/server scope. For a list of all the available configurations and their scope, please refer to [section 5 of the 'man' page] [man5]. ``` bash man 5 barman ``` ## Examples of configuration The following is a basic example of main configuration file: ``` ini [barman] barman_user = barman configuration_files_directory = /etc/barman.d barman_home = /var/lib/barman log_file = /var/log/barman/barman.log log_level = INFO compression = gzip ``` The example below, on the other hand, is a server configuration file that uses streaming backup: ``` ini [streaming-pg] description = "Example of PostgreSQL Database (Streaming-Only)" conninfo = host=pg user=barman dbname=postgres streaming_conninfo = host=pg user=streaming_barman backup_method = postgres streaming_archiver = on slot_name = barman ``` The following code shows a basic example of traditional backup using `rsync`/SSH: ``` ini [ssh-pg] description = "Example of PostgreSQL Database (via Ssh)" ssh_command = ssh postgres@pg conninfo = host=pg user=barman dbname=postgres backup_method = rsync parallel_jobs = 1 reuse_backup = link archiver = on ``` For more detailed information, please refer to the distributed `barman.conf` file, as well as the `ssh-server.conf-template` and `streaming-server.conf-template` template files. barman-2.3/doc/manual/20-server_setup.en.md0000644000076500000240000000067313066203575021302 0ustar mnenciastaff00000000000000\newpage # Setup of a new server in Barman As mentioned in the _"Design and architecture"_ section, we will use the following conventions: - `pg` as server ID and host name where PostgreSQL is installed - `backup` as host name where Barman is located - `barman` as the user running Barman on the `backup` server (identified by the parameter `barman_user` in the configuration) - `postgres` as the user running PostgreSQL on the `pg` server barman-2.3/doc/manual/21-preliminary_steps.en.md0000644000076500000240000001712613151271540022317 0ustar mnenciastaff00000000000000## Preliminary steps This section contains some preliminary steps that you need to undertake before setting up your PostgreSQL server in Barman. > **IMPORTANT:** > Before you proceed, it is important that you have made your decision > in terms of WAL archiving and backup strategies, as outlined in the > _"Design and architecture"_ section. In particular, you should > decide which WAL archiving methods to use, as well as the backup > method. ### PostgreSQL connection You need to make sure that the `backup` server can connect to the PostgreSQL server on `pg` as superuser. This operation is mandatory. We recommend creating a specific user in PostgreSQL, named `barman`, as follows: ``` bash postgres@pg$ createuser -s -P barman ``` > **IMPORTANT:** The above command will prompt for a password, > which you are then advised to add to the `~barman/.pgpass` file > on the `backup` server. For further information, please refer to > ["The Password File" section in the PostgreSQL Documentation] [pgpass]. This connection is required by Barman in order to coordinate its activities with the server, as well as for monitoring purposes. You can choose your favourite client authentication method among those offered by PostgreSQL. More information can be found in the ["Client Authentication" section of the PostgreSQL Documentation] [pghba]. Make sure you test the following command before proceeding: ``` bash barman@backup$ psql -c 'SELECT version()' -U barman -h pg postgres ``` Write down the above information (user name, host name and database name) and keep it for later. You will need it with in the `conninfo` option for your server configuration, like in this example: ``` ini [pg] ; ... conninfo = host=pg user=barman dbname=postgres ``` > **NOTE:** Barman honours the `application_name` connection option > for PostgreSQL servers 9.0 or higher. ### PostgreSQL WAL archiving and replication Before you proceed, you need to properly configure PostgreSQL on `pg` to accept streaming replication connections from the Barman server. Please read the following sections in the PostgreSQL documentation: - [Role attributes] [roles] - [The pg_hba.conf file] [authpghba] - [Setting up standby servers using streaming replication] [streamprot] One configuration parameter that is crucially important is the `wal_level` parameter. This parameter must be configured to ensure that all the useful information necessary for a backup to be coherent are included in the transaction log file. ``` ini wal_level = 'replica' ``` For PostgreSQL versions older than 9.6, `wal_level` must be set to `hot_standby`. Restart the PostgreSQL server for the configuration to be refreshed. ### PostgreSQL streaming connection If you plan to use WAL streaming or streaming backup, you need to setup a streaming connection. We recommend creating a specific user in PostgreSQL, named `streaming_barman`, as follows: ``` bash postgres@pg$ createuser -P --replication streaming_barman ``` > **IMPORTANT:** The above command will prompt for a password, > which you are then advised to add to the `~barman/.pgpass` file > on the `backup` server. For further information, please refer to > ["The Password File" section in the PostgreSQL Documentation] [pgpass]. You can manually verify that the streaming connection works through the following command: ``` bash barman@backup$ psql -U streaming_barman -h pg \ -c "IDENTIFY_SYSTEM" \ replication=1 ``` > **IMPORTANT:** > Please make sure you are able to connect via streaming replication > before going any further. You also need to configure the `max_wal_senders` parameter in the PostgreSQL configuration file: ``` ini max_wal_senders = 2 ``` This option represents the maximum number of concurrent streaming connections that the server will be allowed to manage. Another important parameter is `max_replication_slots`, which represents the maximum number of replication slots [^replslot94] that the server will be allowed to manage. This parameter is needed if you are planning to use the streaming connection to receive WAL files over the streaming connection: ``` ini max_replication_slots = 2 ``` [^replslot94]: Replication slots have been introduced in PostgreSQL 9.4. See section _"WAL Streaming / Replication slots"_ for details. The values proposed for `max_replication_slots` and `max_wal_senders` must be considered as examples, and the values you will use in your actual setup must be choosen after a careful evaluation of the architecture. Please consult the PostgreSQL documentation for guidelines and clarifications. ### SSH connections SSH is a protocol and a set of tools that allows you to open a remote shell to a remote server and copy files between the server and the local system. You can find more documentation about SSH usage in the article ["SSH Essentials"][ssh_essentials] by Digital Ocean. SSH key exchange is a very common practice that is used to implement secure passwordless connections between users on different machines, and it's needed to use `rsync` for WAL archiving and for backups. > **NOTE:** > This procedure is not needed if you plan to use the streaming > connection only to archive transaction logs and backup your PostgreSQL > server. [ssh_essentials]: https://www.digitalocean.com/community/tutorials/ssh-essentials-working-with-ssh-servers-clients-and-keys #### SSH configuration of postgres user Unless you have done it before, you need to create an SSH key for the PostgreSQL user. Log in as `postgres`, in the `pg` host and type: ``` bash postgres@pg$ ssh-keygen -t rsa ``` As this key must be used to connect from hosts without providing a password, no passphrase should be entered during the key pair creation. #### SSH configuration of barman user As in the previous paragraph, you need to create an SSH key for the Barman user. Log in as `barman` in the `backup` host and type: ``` bash barman@backup$ ssh-keygen -t rsa ``` For the same reason, no passphrase should be entered. #### From PostgreSQL to Barman The SSH connection from the PostgreSQL server to the backup server is needed to correctly archive WAL files using the `archive_command` setting. To successfully connect from the PostgreSQL server to the backup server, the PostgreSQL public key has to be configured into the authorized keys of the backup server for the `barman` user. The public key to be authorized is stored inside the `postgres` user home directory in a file named `.ssh/id_rsa.pub`, and its content should be included in a file named `.ssh/authorized_keys` inside the home directory of the `barman` user in the backup server. If the `authorized_keys` file doesn't exist, create it using `600` as permissions. The following command should succeed without any output if the SSH key pair exchange has been completed successfully: ``` bash postgres@pg$ ssh barman@backup -C true ``` The value of the `archive_command` configuration parameter will be discussed in the _"WAL archiving via archive_command section"_. #### From Barman to PostgreSQL The SSH connection between the backup server and the PostgreSQL server is used for the traditional backup over rsync. Just as with the connection from the PostgreSQL server to the backup server, we should authorize the public key of the backup server in the PostgreSQL server for the `postgres` user. The content of the file `.ssh/id_rsa.pub` in the `barman` server should be put in the file named `.ssh/authorized_keys` in the PostgreSQL server. The permissions of that file should be `600`. The following command should succeed without any output if the key pair exchange has been completed successfully. ``` bash barman@backup$ ssh postgres@pg -C true ``` barman-2.3/doc/manual/22-config_file.en.md0000644000076500000240000000153713066203575021022 0ustar mnenciastaff00000000000000## The server configuration file Create a new file, called `pg.conf`, in `/etc/barman.d` directory, with the following content: ``` ini [pg] description = "Our main PostgreSQL server" conninfo = host=pg user=barman dbname=postgres backup_method = postgres # backup_method = rsync ``` The `conninfo` option is set accordingly to the section _"Preliminary steps: PostgreSQL connection"_. The meaning of the `backup_method` option will be covered in the backup section of this guide. If you plan to use the streaming connection for WAL archiving or to create a backup of your server, you also need a `streaming_conninfo` parameter in your server configuration file: ``` ini streaming_conninfo = host=pg user=streaming_barman dbname=postgres ``` This value must be choosen accordingly as described in the section _"Preliminary steps: PostgreSQL connection"_. barman-2.3/doc/manual/23-wal_streaming.en.md0000644000076500000240000000723013066203575021407 0ustar mnenciastaff00000000000000## WAL streaming Barman can reduce the Recovery Point Objective (RPO) by allowing users to add continuous WAL streaming from a PostgreSQL server, on top of the standard `archive_command` strategy Barman relies on [`pg_receivexlog`] [25], a utility that has been available from PostgreSQL 9.2 which exploits the native streaming replication protocol and continuously receives transaction logs from a PostgreSQL server (master or standby). > **IMPORTANT:** > Barman requires that `pg_receivexlog` is installed on the same > server. For PostgreSQL 9.2 servers, you need `pg_receivexlog` of > version 9.2 installed alongside Barman. For PostgreSQL 9.3 and > above, it is recommended to install the latest available version of > `pg_receivexlog`, as it is back compatible. Otherwise, users can > install multiple versions of `pg_receivexlog` on the Barman server > and properly point to the specific version for a server, using the > `path_prefix` option in the configuration file. In order to enable streaming of transaction logs, you need to: 1. setup a streaming connection as previously described 2. set the `streaming_archiver` option to `on` The `cron` command, if the aforementioned requirements are met, transparently manages log streaming through the execution of the `receive-wal` command. This is the recommended scenario. However, users can manually execute the `receive-wal` command: ``` bash barman receive-wal ``` > **NOTE:** > The `receive-wal` command is a foreground process. Transaction logs are streamed directly in the directory specified by the `streaming_wals_directory` configuration option and are then archived by the `archive-wal` command. Unless otherwise specified in the `streaming_archiver_name` parameter, and only for PostgreSQL 9.3 or above, Barman will set `application_name` of the WAL streamer process to `barman_receive_wal`, allowing you to monitor its status in the `pg_stat_replication` system view of the PostgreSQL server. ### Replication slots > **IMPORTANT:** replication slots are available since PostgreSQL 9.4 Replication slots are an automated way to ensure that the PostgreSQL server will not remove WAL files until they were received by all archivers. Barman uses this mechanism to receive the transaction logs from PostgreSQL. You can find more information about replication slots in the [PostgreSQL manual][replication-slots]. You can even base your backup architecture on streaming connection only. This scenario is useful to configure Docker-based PostgreSQL servers and even to work with PostgreSQL servers running on Windows. > **IMPORTANT:** > In this moment, the Windows support is still experimental, as it is > not yet part of our continuous integration system. ### How to configure the WAL streaming First, the PostgreSQL server must be configured to stream the transaction log files to the Barman server. To configure the streaming connection from Barman to the PostgreSQL server you need to enable the `streaming_archiver`, as already said, including this line in the server configuration file: ``` ini streaming_archiver = on ``` If you plan to use replication slots (recommended), another essential option for the setup of the streaming-based transaction log archiving is the `slot_name` option: ``` ini slot_name = barman ``` This option defines the name of the replication slot that will be used by Barman. It is mandatory if you want to use replication slots. When you configure the replication slot name, you can create a replication slot for Barman with this command: ``` bash barman@backup$ barman receive-wal --create-slot pg Creating physical replication slot 'barman' on server 'pg' Replication slot 'barman' created ``` barman-2.3/doc/manual/24-wal_archiving.en.md0000644000076500000240000000370713152223257021371 0ustar mnenciastaff00000000000000## WAL archiving via `archive_command` The `archive_command` is the traditional method to archive WAL files. The value of this PostgreSQL configuration parameter must be a shell command to be executed by the PostgreSQL server to copy the WAL files to the Barman incoming directory. You can retrieve the incoming WALs directory using the `show-server` Barman command and looking for the `incoming_wals_directory` value: ``` bash barman@backup$ barman show-server pg |grep incoming_wals_directory incoming_wals_directory: /var/lib/barman/pg/incoming ``` > **IMPORTANT:** > PostgreSQL 9.5 introduced support for WAL file archiving using > `archive_command` from a standby. This feature is not yet implemented > in Barman. Edit the `postgresql.conf` file of the PostgreSQL instance on the `pg` database and activate the archive mode: ``` ini archive_mode = on wal_level = 'replica' archive_command = 'rsync -a %p barman@backup:INCOMING_WALS_DIRECTORY/%f' ``` Make sure you change the `INCOMING_WALS_DIRECTORY` placeholder with the value returned by the `barman show-server pg` command above. Restart the PostgreSQL server. In order to test that continuous archiving is on and properly working, you need to check both the PostgreSQL server and the backup server. In particular, you need to check that WAL files are correctly collected in the destination directory. ## Verification of WAL archiving configuration In order to improve the verification of the WAL archiving process, the `switch-wal` command has been developed: ``` bash barman@backup$ barman switch-wal --force --archive pg ``` The above command will force PostgreSQL to switch WAL file and trigger the archiving process in Barman. Barman will wait for one file to arrive within 30 seconds (you can change the timeout through the `--archive-timeout` option). If no WAL file is received, an error is returned. You can verify if the WAL archiving has been correctly configured using the `barman check` command. barman-2.3/doc/manual/25-streaming_backup.en.md0000644000076500000240000000251413066203575022073 0ustar mnenciastaff00000000000000## Streaming backup Barman can backup a PostgreSQL server using the streaming connection, relying on `pg_basebackup`, a utility that has been available from PostgreSQL 9.1. > **IMPORTANT:** Barman requires that `pg_basebackup` is installed in > the same server. For PostgreSQL 9.2 servers, you need the > `pg_basebackup` of version 9.2 installed alongside with Barman. For > PostgreSQL 9.3 and above, it is recommented to install the last > available version of `pg_basebackup`, as it is back compatible. You > can even install multiple versions of `pg_basebackup` on the Barman > server and properly point to the specific version for a server, > using the `path_prefix` option in the configuration file. To successfully backup your server with the streaming connection, you need to use `postgres` as your backup method: ``` ini backup_method = postgres ``` > **IMPORTANT:** keep in mind that if the WAL archiving is not > currently configured, you will not be able to start a backup. To check if the server configuration is valid you can use the `barman check` command: ``` bash barman@backup$ barman check pg ``` To start a backup you can use the `barman backup` command: ``` bash barman@backup$ barman backup pg ``` > **IMPORTANT:** `pg_basebackup` 9.4 or higher is required for > tablespace support if you use the `postgres` backup method. barman-2.3/doc/manual/26-rsync_backup.en.md0000644000076500000240000000172013066203575021237 0ustar mnenciastaff00000000000000## Backup with `rsync`/SSH The backup over `rsync` was the only available method before 2.0, and is currently the only backup method that supports the incremental backup feature. Please consult the _"Features in detail"_ section for more information. To take a backup using `rsync` you need to put these parameters inside the Barman server configuration file: ``` ini backup_method = rsync ssh_command = ssh postgres@pg ``` The `backup_method` option activates the `rsync` backup method, and the `ssh_command` option is needed to correctly create an SSH connection from the Barman server to the PostgreSQL server. > **IMPORTANT:** Keep in mind that if the WAL archiving is not > currently configured, you will not be able to start a backup. To check if the server configuration is valid you can use the `barman check` command: ``` bash barman@backup$ barman check pg ``` To take a backup use the `barman backup` command: ``` bash barman@backup$ barman backup pg ``` barman-2.3/doc/manual/27-windows-support.en.md0000644000076500000240000000246713066203575021772 0ustar mnenciastaff00000000000000## How to setup a Windows based server You can backup a PostgreSQL server running on Windows using the streaming connection for both WAL archiving and for backups. > **IMPORTANT:** This feature is still experimental because it is not > yet part of our continuous integration system. Follow every step discussed previously for a streaming connection setup. > **WARNING:**: At this moment, `pg_basebackup` interoperability from > Windows to Linux is still experimental. If you are having issues > taking a backup from a Windows server and your PostgreSQL locale is > not in English, a possible workaround for the issue is instructing > your PostgreSQL to emit messages in English. You can do this by > putting the following parameter in your `postgresql.conf` file: > > ``` ini > lc_messages = 'English' > ``` > > This has been reported to fix the issue. You can backup your server as usual. Remote recovery is not supported for Windows servers, so you must recover your cluster locally in the Barman server and then copy all the files on a Windows server or use a folder shared between the PostgreSQL server and the Barman server. Additionally, make sure that the system user chosen to run PostgreSQL has the permission needed to access the restored data. Basically, it must have full control over the PostgreSQL data directory. barman-2.3/doc/manual/41-global-commands.en.md0000644000076500000240000000446013110540143021575 0ustar mnenciastaff00000000000000\newpage # General commands Barman has many commands and, for the sake of exposition, we can organize them by scope. The scope of the **general commands** is the entire Barman server, that can backup many PostgreSQL servers. **Server commands**, instead, act only on a specified server. **Backup commands** work on a backup, which is taken from a certain server. The following list includes the general commands. ## `cron` `barman` doesn't include a long-running daemon or service file (there's nothing to `systemctl start`, `service start`, etc.). Instead, the `barman cron` subcommand is provided to perform `barman`'s background "steady-state" backup operations. You can perform maintenance operations, on both WAL files and backups, using the `cron` command: ``` bash barman cron ``` > **NOTE:** > This command should be executed in a _cron script_. Our > recommendation is to schedule `barman cron` to run every minute. If > you installed Barman using the rpm or debian package, a cron entry > running on every minute will be created for you. `barman cron` executes WAL archiving operations concurrently on a server basis, and this also enforces retention policies on those servers that have: - `retention_policy` not empty and valid; - `retention_policy_mode` set to `auto`. The `cron` command ensures that WAL streaming is started for those servers that have requested it, by transparently executing the `receive-wal` command. In order to stop the operations started by the `cron` command, comment out the cron entry and execute: ```bash barman receive-wal --stop SERVER_NAME ``` You might want to check `barman list-server` to make sure you get all of your servers. ## `diagnose` The `diagnose` command creates a JSON report useful for diagnostic and support purposes. This report contains information for all configured servers. > **IMPORTANT:** > Even if the diagnose is written in JSON and that format is thought > to be machine readable, its structure is not to be considered part > of the interface. Format can change between different Barman versions. ## `list-server` You can display the list of active servers that have been configured for your backup system with: ``` bash barman list-server ``` A machine readble output can be obtained with the `--minimal` option: ``` bash barman list-server --minimal ``` barman-2.3/doc/manual/42-server-commands.en.md0000644000076500000240000002024313152223257021653 0ustar mnenciastaff00000000000000\newpage # Server commands As we said in the previous section, server commands work directly on a PostgreSQL server or on its area in Barman, and are useful to check its status, perform maintainance operations, take backups, and manage the WAL archive. ## `archive_wal` The `archive_wal` command execute maintainance operations on WAL files for a given server. This operations include processing of the WAL files received from the streaming connection or from the `archive_command` or both. > **IMPORTANT:** > The `archive_wal` command, even if it can be directly invoked, is > designed to be started from the `cron` general command. ## `backup` The `backup` command takes a full backup (_base backup_) of a given server. It has several options that let you override the corresponding configuration parameter for the new backup. For more information, consult the manual page. You can perform a full backup for a given server with: ``` bash barman backup ``` > **TIP:** > You can use `barman backup all` to sequentially backup all your > configured servers. ## `check` You can check the connection to a given server and the configuration coherence with the `check` command: ``` bash barman check ``` > **TIP:** > You can use `barman check all` to check all your configured servers. > **IMPORTANT:** > The `check` command is probably the most critical feature that > Barman implements. We recommend to integrate it with your alerting > and monitoring infrastructure. The `--nagios` option allows you > to easily create a plugin for Nagios/Icinga. ## `get-wal` Barman allows users to request any _xlog_ file from its WAL archive through the `get-wal` command: ``` bash barman get-wal [-o OUTPUT_DIRECTORY] [-j|-x] ``` If the requested WAL file is found in the server archive, the uncompressed content will be returned to `STDOUT`, unless otherwise specified. The following options are available for the `get-wal` command: - `-o` allows users to specify a destination directory where Barman will deposit the requested WAL file - `-j` will compress the output using `bzip2` algorithm - `-x` will compress the output using `gzip` algorithm - `-p SIZE` peeks from the archive up to WAL files, starting from the requested file It is possible to use `get-wal` during a recovery operation, transforming the Barman server into a _WAL hub_ for your servers. This can be automatically achieved by adding the `get-wal` value to the `recovery_options` global/server configuration option: ``` ini recovery_options = 'get-wal' ``` `recovery_options` is a global/server option that accepts a list of comma separated values. If the keyword `get-wal` is present during a recovery operation, Barman will prepare the `recovery.conf` file by setting the `restore_command` so that `barman get-wal` is used to fetch the required WAL files. Similarly, one can use the `--get-wal` option for the `recover` command at run-time. This is an example of a `restore_command` for a local recovery: ``` ini restore_command = 'sudo -u barman barman get-wal SERVER %f > %p' ``` Please note that the `get-wal` command should always be invoked as `barman` user, and that it requires the correct permission to read the WAL files from the catalog. This is the reason why we are using `sudo -u barman` in the example. Setting `recovery_options` to `get-wal` for a remote recovery will instead generate a `restore_command` using the `barman-wal-restore` script. `barman-wal-restore` is a more resilient shell script which manages SSH connection errors. This script has many useful options such as the automatic compression and decompression of the WAL files and the *peek* feature, which allows you to retrieve the next WAL files while PostgreSQL is applying one of them. It is an excellent way to optimise the bandwidth usage between PostgreSQL and Barman. `barman-wal-restore` is available in the `barman-cli` project or package. This is an example of a `restore_command` for a remote recovery: ``` ini restore_command = 'barman-wal-restore -U barman backup SERVER %f %p' ``` Since it uses SSH to communicate with the Barman server, SSH key authentication is required for the `postgres` user to login as `barman` on the backup server. > **IMPORTANT:** > Even though `recovery_options` aims to automate the process, using > the `get-wal` facility requires manual intervention and proper > testing. ## `list-backup` You can list the catalog of available backups for a given server with: ``` bash barman list-backup ``` > **TIP:** You can request a full list of the backups of all servers > using `all` as the server name. To have a machine-readable output you can use the `--minimal` option. ## `rebuild-xlogdb` At any time, you can regenerate the content of the WAL archive for a specific server (or every server, using the `all` shortcut). The WAL archive is contained in the `xlog.db` file and every server managed by Barman has its own copy. The `xlog.db` file can be rebuilt with the `rebuild-xlogdb` command. This will scan all the archived WAL files and regenerate the metadata for the archive. For example: ``` bash barman rebuild-xlogdb ``` ## `receive-wal` This command manages the `receive-wal` process, which uses the streaming protocol to receive WAL files from the PostgreSQL streaming connection. ### receive-wal process management If the command is run without options, a `receive-wal` process will be started. This command is based on the `pg_receivexlog` PostgreSQL command. ``` bash barman receive-wal ``` If the command is run with the `--stop` option, the currently running `receive-wal` process will be stopped. The `receive-wal` process uses a status file to track last written record of the transaction log. When the status file needs to be cleaned, the `--reset` option can be used. > **IMPORTANT:** If you are not using replication slots, you rely > on the value of `wal_keep_segments`. Be aware that under high peeks > of workload on the database, the `receive-wal` process > might fall behind and go out of sync. As a precautionary measure, > Barman currently requires that users manually execute the command with the > `--reset` option, to avoid making wrong assumptions. ### Replication slot management The `receive-wal` process is also useful to create or drop the replication slot needed by Barman for its WAL archiving procedure. With the `--create-slot` option, the replication slot named after the `slot_name` configuration option will be created on the PostgreSQL server. With the `--drop-slot`, the previous replication slot will be deleted. ## `replication-status` The `replication-status` command reports the status of any streaming client currently attached to the PostgreSQL server, including the `receive-wal` process of your Barman server (if configured). You can execute the command as follows: ``` bash barman replication-status ``` > **TIP:** You can request a full status report of the replica > for all your servers using `all` as the server name. To have a machine-readable output you can use the `--minimal` option. ## `show-server` You can show the configuration parameters for a given server with: ``` bash barman show-server ``` > **TIP:** you can request a full configuration report using `all` as > the server name. ## `status` The `status` command shows live information and status of a PostgreSQL server or of all servers if you use `all` as server name. ``` bash barman status ``` ## `switch-wal` This command makes the PostgreSQL server switch to another transaction log file (WAL), allowing the current log file to be closed, received and then archived. ``` bash barman switch-wal ``` If there has been no transaction activity since the last transaction log file switch, the switch needs to be forced using the `--force` option. The `--archive` option requests Barman to trigger WAL archiving after the xlog switch. By default, a 30 seconds timeout is enforced (this can be changed with `--archive-timeout`). If no WAL file is received, an error is returned. > **NOTE:** In Barman 2.1 and 2.2 this command was called `switch-xlog`. > It has been renamed for naming consistency with PostgreSQL 10 and higher. barman-2.3/doc/manual/43-backup-commands.en.md0000644000076500000240000001432413151745346021625 0ustar mnenciastaff00000000000000\newpage # Backup commands Backup commands are those that works directly on backups already existing in Barman's backup catalog. > **NOTE:** > Remember a backup ID can be retrieved with `barman list-backup > ` ## Backup ID shortcuts Barman allows you to use special keywords to identify a specific backup: * `last/latest`: identifies the newest backup in the catalog * `first/oldest`: identifies the oldest backup in the catalog Using those keywords with Barman commands allows you to execute actions without knowing the exact ID of a backup for a server. For example we can issue: ``` bash barman delete oldest ``` to remove the oldest backup available in the catalog and reclaim disk space. ## `delete` You can delete a given backup with: ``` bash barman delete ``` The `delete` command accepts any [shortcut](#shortcuts) to identify backups. ## `list-files` You can list the files (base backup and required WAL files) for a given backup with: ``` bash barman list-files [--target TARGET_TYPE] ``` With the `--target TARGET_TYPE` option, it is possible to choose the content of the list for a given backup. Possible values for `TARGET_TYPE` are: - `data`: lists the data files - `standalone`: lists the base backup files, including required WAL files - `wal`: lists all WAL files from the beginning of the base backup to the start of the following one (or until the end of the log) - `full`: same as `data` + `wal` The default value for `TARGET_TYPE` is `standalone`. > **IMPORTANT:** > The `list-files` command facilitates interaction with external > tools, and can therefore be extremely useful to integrate > Barman into your archiving procedures. ## `recover` The `recover` command is used to recover a whole server after a backup is executed using the `backup` command. This is achieved issuing a command like the following: ```bash barman@backup$ barman recover /path/to/recover/dir ``` At the end of the execution of the recovery, the selected backup is recovered locally and the destination path contains a data directory ready to be used to start a PostgreSQL instance. > **IMPORTANT:** > Running this command as user `barman`, it will become the database superuser. The specific ID of a backup can be retrieved using the [list-backup](#list-backup) command. > **IMPORTANT:** > Barman does not currently keep track of symbolic links inside PGDATA > (except for tablespaces inside pg_tblspc). We encourage > system administrators to keep track of symbolic links and to add them > to the disaster recovery plans/procedures in case they need to be restored > in their original location. The recovery command has several options that modify the command behavior. ### Remote recovery Add the `--remote-ssh-command ` option to the invocation of the recovery command. Doing this will allow Barman to execute the copy on a remote server, using the provided command to connect to the remote host. > **NOTE:** > It is advisable to use the `postgres` user to perform > the recovery on the remote host. Known limitations of the remote recovery are: * Barman requires at least 4GB of free space in the system temporary directory unless the [`get-wal`](#get-wal) command is specified in the `recovery_option` parameter in the Barman configuration. * The SSH connection between Barman and the remote host **must** use the public key exchange authentication method * The remote user **must** be able to create the directory structure of the backup in the destination directory. * There must be enough free space on the remote server to contain the base backup and the WAL files needed for recovery. ### Tablespace remapping Barman is able to automatically remap one or more tablespaces using the recover command with the --tablespace option. The option accepts a pair of values as arguments using the `NAME:DIRECTORY` format: * `NAME` is the identifier of the tablespace * `DIRECTORY` is the new destination path for the tablespace If the destination directory does not exists, Barman will try to create it (assuming you have the required permissions). ### Point in time recovery Barman wraps PostgreSQL's Point-in-Time Recovery (PITR), allowing you to specify a recovery target, either as a timestamp, as a restore label, or as a transaction ID. > **IMPORTANT:** > The earliest PITR for a given backup is the end of the base > backup itself. If you want to recover at any point in time > between the start and the end of a backup, you must use > the previous backup. From Barman 2.3 you can exit recovery > when consistency is reached by using `--target-immediate` option > (available only for PostgreSQL 9.4 and newer). The recovery target can be specified using one of four mutually exclusive options: * `--target-time TARGET_TIME`: to specify a timestamp * `--target-xid TARGET_XID`: to specify a transaction ID * `--target-name TARGET_NAME`: to specify a named restore point previously created with the pg_create_restore_point(name) function[^TARGET_NAME] * `--target-immediate`: recovery ends when a consistent state is reached (that is the end of the base backup process) [^RECOVERY_TARGET_IMMEDIATE] > **IMPORTANT:** > Recovery target via _time_ and _xid_ **must be** subsequent to the > end of the backup. If you want to recover to a point in time between > the start and the end of a backup, you must recover from the > previous backup in the catalogue. [^TARGET_NAME]: Only available on PostgreSQL 9.1 and above [^RECOVERY_TARGET_IMMEDIATE]: Only available on PostgreSQL 9.4 and above You can use the `--exclusive` option to specify whether to stop immediately before or immediately after the recovery target. Barman allows you to specify a target timeline for recovery, using the `target-tli` option. The notion of timeline goes beyond the scope of this document; you can find more details in the PostgreSQL documentation, as mentioned in the _"Before you start"_ section. ## `show-backup` You can retrieve all the available information for a particular backup of a given server with: ``` bash barman show-backup ``` The `show-backup` command accepts any [shortcut](#shortcuts) to identify backups. barman-2.3/doc/manual/50-feature-details.en.md0000644000076500000240000005441713113023325021624 0ustar mnenciastaff00000000000000\newpage # Features in detail In this section we present several Barman features and discuss their applicability and the configuration required to use them. This list is not exhaustive, as many scenarios can be created working on the Barman configuration. Nevertheless, it is useful to discuss common patterns. ## Backup features ### Incremental backup Barman implements **file-level incremental backup**. Incremental backup is a type of full periodic backup which only saves data changes from the latest full backup available in the catalog for a specific PostgreSQL server. It must not be confused with differential backup, which is implemented by _WAL continuous archiving_. > **NOTE:** Block level incremental backup will be available in > future versions. > **IMPORTANT:** The `reuse_backup` option can't be used with the > `postgres` backup method at this time. The main goals of incremental backups in Barman are: - Reduce the time taken for the full backup process - Reduce the disk space occupied by several periodic backups (**data deduplication**) This feature heavily relies on `rsync` and [hard links] [8], which must therefore be supported by both the underlying operating system and the file system where the backup data resides. The main concept is that a subsequent base backup will share those files that have not changed since the previous backup, leading to relevant savings in disk usage. This is particularly true of VLDB contexts and of those databases containing a high percentage of _read-only historical tables_. Barman implements incremental backup through a global/server option called `reuse_backup`, that transparently manages the `barman backup` command. It accepts three values: - `off`: standard full backup (default) - `link`: incremental backup, by reusing the last backup for a server and creating a hard link of the unchanged files (for backup space and time reduction) - `copy`: incremental backup, by reusing the last backup for a server and creating a copy of the unchanged files (just for backup time reduction) The most common scenario is to set `reuse_backup` to `link`, as follows: ``` ini reuse_backup = link ``` Setting this at global level will automatically enable incremental backup for all your servers. As a final note, users can override the setting of the `reuse_backup` option through the `--reuse-backup` runtime option for the `barman backup` command. Similarly, the runtime option accepts three values: `off`, `link` and `copy`. For example, you can run a one-off incremental backup as follows: ``` bash barman backup --reuse-backup=link ``` ### Limiting bandwidth usage It is possible to limit the usage of I/O bandwidth through the `bandwidth_limit` option (global/per server), by specifying the maximum number of kilobytes per second. By default it is set to 0, meaning no limit. > **IMPORTANT:** the `bandwidth_limit` and the > `tablespace_bandwidth_limit` options are not supported with the > `postgres` backup method In case you have several tablespaces and you prefer to limit the I/O workload of your backup procedures on one or more tablespaces, you can use the `tablespace_bandwidth_limit` option (global/per server): ``` ini tablespace_bandwidth_limit = tbname:bwlimit[, tbname:bwlimit, ...] ``` The option accepts a comma separated list of pairs made up of the tablespace name and the bandwidth limit (in kilobytes per second). When backing up a server, Barman will try and locate any existing tablespace in the above option. If found, the specified bandwidth limit will be enforced. If not, the default bandwidth limit for that server will be applied. ### Network Compression It is possible to reduce the size of transferred data using compression. It can be enabled using the `network_compression` option (global/per server): > **IMPORTANT:** the `network_compression` option is not available > with the `postgres` backup method. ``` ini network_compression = true|false ``` Setting this option to `true` will enable data compression during network transfers (for both backup and recovery). By default it is set to `false`. ### Concurrent Backup and backup from a standby Normally, during backup operations, Barman uses PostgreSQL native functions `pg_start_backup` and `pg_stop_backup` for _exclusive backup_. These operations are not allowed on a read-only standby server. Barman is also capable of performing backups of PostgreSQL from 9.2 or greater database servers in a **concurrent way**, primarily through the `backup_options` configuration parameter.[^ABOUT_CONCURRENT_BACKUP] [^ABOUT_CONCURRENT_BACKUP]: Concurrent backup is a technology that has been available in PostgreSQL since version 9.2, through the _streaming replication protocol_ (for example, using a tool like `pg_basebackup`). This introduces a new architecture scenario with Barman: **backup from a standby server**, using `rsync`. > **IMPORTANT:** **Concurrent backup** requires users of PostgreSQL > 9.2, 9.3, 9.4, and 9.5 to install the `pgespresso` open source > extension on every PostgreSQL server of the cluster. For more > detailed information and the source code, please visit the > [pgespresso extension website] [9]. Barman supports the new API > introduced in PostgreSQL 9.6. This removes the requirement of the > `pgespresso` extension to perform concurrent backups from this > version of PostgreSQL. By default, `backup_options` is transparently set to `exclusive_backup` for back compatibility reasons. Users of PostgreSQL 9.6 should set `backup_options` to `concurrent_backup`. When `backup_options` is set to `concurrent_backup`, Barman activates the _concurrent backup mode_ for a server and follows these two simple rules: - `ssh_command` must point to the destination Postgres server - `conninfo` must point to a database on the destination Postgres database. Using PostgreSQL 9.2, 9.3, 9.4, and 9.5, `pgespresso` must be correctly installed through `CREATE EXTENSION`. Using 9.6 or greater, concurrent backups are executed through the Postgres native API. The destination Postgres server can be either the master or a streaming replicated standby server. > **NOTE:** > When backing up from a standby server, continuous archiving of WAL > files must be configured on the master to ship files to the Barman > server (as outlined in the _"WAL archiving via archive_command"_ section > above)[^CONCURRENT_ARCHIVING]. [^CONCURRENT_ARCHIVING]: In case of a concurrent backup, currently Barman has no way to determine that the closing WAL file of a full backup has actually been shipped - opposite of an exclusive backup where PostgreSQL itself makes sure that the WAL file is correctly archived. Be aware that the full backup cannot be considered consistent until that WAL file has been received and archived by Barman. ## Archiving features ### WAL compression The `barman cron` command will compress WAL files if the `compression` option is set in the configuration file. This option allows five values: - `bzip2`: for Bzip2 compression (requires the `bzip2` utility) - `gzip`: for Gzip compression (requires the `gzip` utility) - `pybzip2`: for Bzip2 compression (uses Python's internal compression module) - `pygzip`: for Gzip compression (uses Python's internal compression module) - `pigz`: for Pigz compression (requires the `pigz` utility) - `custom`: for custom compression, which requires you to set the following options as well: - `custom_compression_filter`: a compression filter - `custom_decompression_filter`: a decompression filter > *NOTE:* All methods but `pybzip2` and `pygzip` require `barman > archive-wal` to fork a new process. ### Synchronous WAL streaming > **IMPORTANT:** This feature is available only from PostgreSQL 9.5 > and above. Barman can also reduce the Recovery Point Objective to zero, by collecting the transaction WAL files like a synchronous standby server would. To configure such a scenario, the Barman server must be configured to archive WALs via the [streaming connection](#streaming_connection), and the `receive-wal` process should figure as a synchronous standby of the PostgreSQL server. First of all, you need to retrieve the application name of the Barman `receive-wal` process with the `show-server` command: ``` bash barman@backup$ barman show-server pg|grep streaming_archiver_name streaming_archiver_name: barman_receive_wal ``` Then the application name should be added to the `postgresql.conf` file as a synchronous standby: ``` ini synchronous_standby_names = 'barman_receive_wal' ``` > **IMPORTANT:** this is only an example of configuration, to show you that > Barman is eligible to be a synchronous standby node. > We are not suggesting to use ONLY Barman. > You can read _["Synchronous Replication"][synch]_ from the PostgreSQL > documentation for further information on this topic. The PostgreSQL server needs to be restarted for the configuration to be reloaded. If the server has been configured correctly, the `replication-status` command should show the `receive_wal` process as a synchronous streaming client: ``` bash [root@backup ~]# barman replication-status pg Status of streaming clients for server 'pg': Current xlog location on master: 0/9000098 Number of streaming clients: 1 1. #1 Sync WAL streamer Application name: barman_receive_wal Sync stage : 3/3 Remote write Communication : TCP/IP IP Address : 139.59.135.32 / Port: 58262 / Host: - User name : streaming_barman Current state : streaming (sync) Replication slot: barman WAL sender PID : 2501 Started at : 2016-09-16 10:33:01.725883+00:00 Sent location : 0/9000098 (diff: 0 B) Write location : 0/9000098 (diff: 0 B) Flush location : 0/9000098 (diff: 0 B) ``` ## Catalog management features ### Minimum redundancy safety You can define the minimum number of periodic backups for a PostgreSQL server, using the global/per server configuration option called `minimum_redundancy`, by default set to 0. By setting this value to any number greater than 0, Barman makes sure that at any time you will have at least that number of backups in a server catalog. This will protect you from accidental `barman delete` operations. > **IMPORTANT:** > Make sure that your retention policy settings do not collide with > minimum redundancy requirements. Regularly check Barman's log for > messages on this topic. ### Retention policies Barman supports **retention policies** for backups. A backup retention policy is a user-defined policy that determines how long backups and related archive logs (Write Ahead Log segments) need to be retained for recovery procedures. Based on the user's request, Barman retains the periodic backups required to satisfy the current retention policy and any archived WAL files required for the complete recovery of those backups. Barman users can define a retention policy in terms of **backup redundancy** (how many periodic backups) or a **recovery window** (how long). Retention policy based on redundancy : In a redundancy based retention policy, the user determines how many periodic backups to keep. A redundancy-based retention policy is contrasted with retention policies that use a recovery window. Retention policy based on recovery window : A recovery window is one type of Barman backup retention policy, in which the DBA specifies a period of time and Barman ensures retention of backups and/or archived WAL files required for point-in-time recovery to any time during the recovery window. The interval always ends with the current time and extends back in time for the number of days specified by the user. For example, if the retention policy is set for a recovery window of seven days, and the current time is 9:30 AM on Friday, Barman retains the backups required to allow point-in-time recovery back to 9:30 AM on the previous Friday. #### Scope Retention policies can be defined for: - **PostgreSQL periodic base backups**: through the `retention_policy` configuration option - **Archive logs**, for Point-In-Time-Recovery: through the `wal_retention_policy` configuration option > **IMPORTANT:** > In a temporal dimension, archive logs must be included in the time > window of periodic backups. There are two typical use cases here: full or partial point-in-time recovery. Full point in time recovery scenario: : Base backups and archive logs share the same retention policy, allowing you to recover at any point in time from the first available backup. Partial point in time recovery scenario: : Base backup retention policy is wider than that of archive logs, for example allowing users to keep full, weekly backups of the last 6 months, but archive logs for the last 4 weeks (granting to recover at any point in time starting from the last 4 periodic weekly backups). > **IMPORTANT:** > Currently, Barman implements only the **full point in time > recovery** scenario, by constraining the `wal_retention_policy` > option to `main`. #### How they work Retention policies in Barman can be: - **automated**: enforced by `barman cron` - **manual**: Barman simply reports obsolete backups and allows you to delete them > **IMPORTANT:** > Currently Barman does not implement manual enforcement. This feature > will be available in future versions. #### Configuration and syntax Retention policies can be defined through the following configuration options: - `retention_policy`: for base backup retention - `wal_retention_policy`: for archive logs retention - `retention_policy_mode`: can only be set to `auto` (retention policies are automatically enforced by the `barman cron` command) These configuration options can be defined both at a global level and a server level, allowing users maximum flexibility on a multi-server environment. ##### Syntax for `retention_policy` The general syntax for a base backup retention policy through `retention_policy` is the following: ``` ini retention_policy = {REDUNDANCY value | RECOVERY WINDOW OF value {DAYS | WEEKS | MONTHS}} ``` Where: - syntax is case insensitive - `value` is an integer and is > 0 - in case of **redundancy retention policy**: - `value` must be greater than or equal to the server minimum redundancy level (if that value is not assigned, a warning is generated) - the first valid backup is the value-th backup in a reverse ordered time series - in case of **recovery window policy**: - the point of recoverability is: current time - window - the first valid backup is the first available backup before the point of recoverability; its value in a reverse ordered time series must be greater than or equal to the server minimum redundancy level (if it is not assigned to that value and a warning is generated) By default, `retention_policy` is empty (no retention enforced). ##### Syntax for `wal_retention_policy` Currently, the only allowed value for `wal_retention_policy` is the special value `main`, that maps the retention policy of archive logs to that of base backups. ## Hook scripts Barman allows a database administrator to run hook scripts on these two events: - before and after a backup - before and after a WAL file is archived There are two types of hook scripts that Barman can manage: - standard hook scripts - retry hook scripts The only difference between these two types of hook scripts is that Barman executes a standard hook script only once, without checking its return code, whereas a retry hook script may be executed more than once, depending on its return code. Specifically, when executing a retry hook script, Barman checks the return code and retries indefinitely until the script returns either `SUCCESS` (with standard return code `0`), or `ABORT_CONTINUE` (return code `62`), or `ABORT_STOP` (return code `63`). Barman treats any other return code as a transient failure to be retried. Users are given more power: a hook script can control its workflow by specifying whether a failure is transient. Also, in case of a 'pre' hook script, by returning `ABORT_STOP`, users can request Barman to interrupt the main operation with a failure. Hook scripts are executed in the following order: 1. The standard 'pre' hook script (if present) 2. The retry 'pre' hook script (if present) 3. The actual event (i.e. backup operation, or WAL archiving), if retry 'pre' hook script was not aborted with `ABORT_STOP` 4. The retry 'post' hook script (if present) 5. The standard 'post' hook script (if present) The output generated by any hook script is written in the log file of Barman. > **NOTE:** > Currently, `ABORT_STOP` is ignored by retry 'post' hook scripts. In > these cases, apart from lodging an additional warning, `ABORT_STOP` > will behave like `ABORT_CONTINUE`. ### Backup scripts These scripts can be configured with the following global configuration options (which can be overridden on a per server basis): - `pre_backup_script`: _hook script_ executed _before_ a base backup, only once, with no check on the exit code - `pre_backup_retry_script`: _retry hook script_ executed _before_ a base backup, repeatedly until success or abort - `post_backup_retry_script`: _retry hook script_ executed _after_ a base backup, repeatedly until success or abort - `post_backup_script`: _hook script_ executed _after_ a base backup, only once, with no check on the exit code The script definition is passed to a shell and can return any exit code. Only in case of a _retry_ script, Barman checks the return code (see the [hook script section](#hook_scripts)). The shell environment will contain the following variables: - `BARMAN_BACKUP_DIR`: backup destination directory - `BARMAN_BACKUP_ID`: ID of the backup - `BARMAN_CONFIGURATION`: configuration file used by Barman - `BARMAN_ERROR`: error message, if any (only for the `post` phase) - `BARMAN_PHASE`: phase of the script, either `pre` or `post` - `BARMAN_PREVIOUS_ID`: ID of the previous backup (if present) - `BARMAN_RETRY`: `1` if it is a retry script, `0` if not - `BARMAN_SERVER`: name of the server - `BARMAN_STATUS`: status of the backup - `BARMAN_VERSION`: version of Barman ### WAL archive scripts Similar to backup scripts, archive scripts can be configured with global configuration options (which can be overridden on a per server basis): - `pre_archive_script`: _hook script_ executed _before_ a WAL file is archived by maintenance (usually `barman cron`), only once, with no check on the exit code - `pre_archive_retry_script`: _retry hook script_ executed _before_ a WAL file is archived by maintenance (usually `barman cron`), repeatedly until it is successful or aborted - `post_archive_retry_script`: _retry hook script_ executed _after_ a WAL file is archived by maintenance, repeatedly until it is successful or aborted - `post_archive_script`: _hook script_ executed _after_ a WAL file is archived by maintenance, only once, with no check on the exit code The script is executed through a shell and can return any exit code. Only in case of a _retry_ script, Barman checks the return code (see the upper section). Archive scripts share with backup scripts some environmental variables: - `BARMAN_CONFIGURATION`: configuration file used by Barman - `BARMAN_ERROR`: error message, if any (only for the `post` phase) - `BARMAN_PHASE`: phase of the script, either `pre` or `post` - `BARMAN_SERVER`: name of the server Following variables are specific to archive scripts: - `BARMAN_SEGMENT`: name of the WAL file - `BARMAN_FILE`: full path of the WAL file - `BARMAN_SIZE`: size of the WAL file - `BARMAN_TIMESTAMP`: WAL file timestamp - `BARMAN_COMPRESSION`: type of compression used for the WAL file ## Customization ### Lock file directory Barman allows you to specify a directory for lock files through the `barman_lock_directory` global option. Lock files are used to coordinate concurrent work at global and server level (for example, cron operations, backup operations, access to the WAL archive, and so on.). By default (for backward compatibility reasons), `barman_lock_directory` is set to `barman_home`. > **TIP:** > Users are encouraged to use a directory in a volatile partition, > such as the one dedicated to run-time variable data (e.g. > `/var/run/barman`). ### Binary paths As of version 1.6.0, Barman allows users to specify one or more directories where Barman looks for executable files, using the global/server option `path_prefix`. If a `path_prefix` is provided, it must contain a list of one or more directories separated by colon. Barman will search inside these directories first, then in those specified by the `PATH` environment variable. By default the `path_prefix` option is empty. ## Integration with cluster management systems Barman has been designed for integration with standby servers (with streaming replication or traditional file based log shipping) and high availability tools like [repmgr] [repmgr]. From an architectural point of view, PostgreSQL must be configured to archive WAL files directly to the Barman server. Barman, thanks to the `get-wal` framework, can also be used as a WAL hub. For this purpose, you can use the `barman-wal-restore` script, part of the `barman-cli` package, with all your standby servers. The `replication-status` command allows you to get information about any streaming client attached to the managed server, in particular hot standby servers and WAL streamers. ## Parallel jobs By default, Barman uses only one worker for file copy during both backup and recover operations. Starting from version 2.2, it is possible to customize the number of workers that will perform file copy. In this case, the files to be copied will be equally distributed among all parallel workers. It can be configured in global and server scopes, adding these in the corresponding configuration file: ``` ini parallel_jobs = n ``` where `n` is the desired number of parallel workers to be used in file copy operations. The default value is 1. In any case, users can override this value at run-time when executing `backup` or `recover` commands. For example, you can use 4 parallel workers as follows: ``` bash barman backup --jobs 4 server1 ``` Or, alternatively: ``` bash barman backup --j 4 server1 ``` Please note that this parallel jobs feature is only available for servers configured through `rsync`/SSH. For servers configured through streaming protocol, Barman will rely on `pg_basebackup` which is currently limited to only one worker. barman-2.3/doc/manual/65-troubleshooting.en.md0000644000076500000240000000330513066203575022007 0ustar mnenciastaff00000000000000\newpage # Troubleshooting ## Diagnose a Barman installation You can gather important information about the status of all the configured servers using: ``` bash barman diagnose ``` The `diagnose` command output is a full snapshot of the barman server, providing useful information, such as global configuration, SSH version, Python version, `rsync` version, PostgreSQL clients version, as well as current configuration and status of all servers. The `diagnose` command is extremely useful for troubleshooting problems, as it gives a global view on the status of your Barman installation. ## Requesting help Although Barman is extensively documented, there are a lot of scenarios that are not covered. For any questions about Barman and disaster recovery scenarios using Barman, you can reach the dev team using the community mailing list: https://groups.google.com/group/pgbarman or the IRC channel on freenode: irc://irc.freenode.net/barman In the event you discover a bug, you can open a ticket using Github: https://github.com/2ndquadrant-it/barman/issues 2ndQuadrant provides professional support for Barman, including 24/7 service. ### Submitting a bug Barman has been extensively tested and is currently being used in several production environments. However, as any software, Barman is not bug free. If you discover a bug, please follow this procedure: - execute the `barman diagnose` command - file a bug through the Github issue tracker, by attaching the output obtained by the diagnostics command above (`barman diagnose`) > **WARNING:** > Be careful when submitting the output of the diagnose command > as it might disclose information that are potentially dangerous > from a security point of view. barman-2.3/doc/manual/66-about.en.md0000644000076500000240000000726013134472625017677 0ustar mnenciastaff00000000000000\newpage # The Barman project ## Support and sponsor opportunities Barman is free software, written and maintained by 2ndQuadrant. If you require support on using Barman, or if you need new features, please get in touch with 2ndQuadrant. You can sponsor the development of new features of Barman and PostgreSQL which will be made publicly available as open source. For further information, please visit: - [Barman website] [11] - [Support section] [12] - [2ndQuadrant website] [13] - [Barman FAQs] [14] - [2ndQuadrant blog: Barman] [15] ## Contributing to Barman 2ndQuadrant has a team of software engineers, architects, database administrators, system administrators, QA engineers, developers and managers that dedicate their time and expertise to improve Barman's code. We adopt lean and agile methodologies for software development, and we believe in the _devops_ culture that allowed us to implement rigorous testing procedures through cross-functional collaboration. Every Barman commit is the contribution of multiple individuals, at different stages of the production pipeline. Even though this is our preferred way of developing Barman, we gladly accept patches from external developers, as long as: - user documentation (tutorial and man pages) is provided. - source code is properly documented and contains relevant comments. - code supplied is covered by unit tests. - no unrelated feature is compromised or broken. - source code is rebased on the current master branch. - commits and pull requests are limited to a single feature (multi-feature patches are hard to test and review). - changes to the user interface are discussed beforehand with 2ndQuadrant. We also require that any contributions provide a copyright assignment and a disclaimer of any work-for-hire ownership claims from the employer of the developer. You can use Github's pull requests system for this purpose. ## Authors In alphabetical order: - Gabriele Bartolini, (architect) - Jonathan Battiato, (QA/testing) - Giulio Calacoci, (developer) - Francesco Canovai, (QA/testing) - Leonardo Cecchi, (developer) - Gianni Ciolli, (QA/testing) - Britt Cole, (documentation) - Marco Nenciarini, (project leader) - Rubens Souza, (QA/testing) Past contributors: - Carlo Ascani - Stefano Bianucci - Giuseppe Broccolo ## Links - [check-barman] [16]: a Nagios plugin for Barman, written by Holger Hamann (MIT license) - [puppet-barman] [17]: Barman module for Puppet (GPL) - [Tutorial on "How To Back Up, Restore, and Migrate PostgreSQL Databases with Barman on CentOS 7"] [26], by Sadequl Hussain (available on DigitalOcean Community) - [BarmanAPI] [27]: RESTFul API for Barman, written by Mehmet Emin Karakaş (GPL) ## License and Contributions Barman is the property of 2ndQuadrant Limited and its code is distributed under GNU General Public License 3. Copyright (C) 2011-2017 [2ndQuadrant Limited] [13]. Barman has been partially funded through [4CaaSt] [18], a research project funded by the European Commission's Seventh Framework programme. Contributions to Barman are welcome, and will be listed in the `AUTHORS` file. 2ndQuadrant Limited requires that any contributions provide a copyright assignment and a disclaimer of any work-for-hire ownership claims from the employer of the developer. This lets us make sure that all of the Barman distribution remains free code. Please contact info@2ndQuadrant.com for a copy of the relevant Copyright Assignment Form. barman-2.3/doc/manual/70-feature-matrix.en.md0000644000076500000240000000361713066203575021517 0ustar mnenciastaff00000000000000\newpage \appendix # Feature matrix Below you will find a matrix of PostgreSQL versions and Barman features for backup and archiving: | **Version** | **Backup with rsync/SSH** | **Backup with pg_basebackup** | **Standard WAL archiving** | **WAL Streaming** | **RPO=0** | |:---------:|:---------------------:|:-------------------------:|:----------------------:|:----------------------:|:-------:| | **9.6** | Yes | Yes | Yes | Yes | Yes | | **9.5** | Yes | Yes | Yes | Yes | Yes ~(d)~ | | **9.4** | Yes | Yes | Yes | Yes | Yes ~(d)~ | | **9.3** | Yes | Yes ~(c)~ | Yes | Yes ~(b)~ | No | | **9.2** | Yes | Yes ~(a)~~(c)~ | Yes | Yes ~(a)~~(b)~ | No | | _9.1_ | Yes | No | Yes | No | No | | _9.0_ | Yes | No | Yes | No | No | | _8.4_ | Yes | No | Yes | No | No | | _8.3_ | Yes | No | Yes | No | No | **NOTE:** a) `pg_basebackup` and `pg_receivexlog` 9.2 required b) WAL streaming-only not supported (standard archiving required) c) Backup of tablespaces not supported d) When using `pg_receivexlog` 9.5, minor version 9.5.5 or higher required [^commitsync] [^commitsync]: The commit ["Fix pg_receivexlog --synchronous"] [49340627f9821e447f135455d942f7d5e96cae6d] is required (included in version 9.5.5) It is required by Barman that `pg_basebackup` and `pg_receivexlog` of the same version of the PostgreSQL server (or higher) are installed on the same server where Barman resides. The only exception is that PostgreSQL 9.2 users are required to install version 9.2 of `pg_basebackup` and `pg_receivexlog` alongside with Barman. >> **TIP:** We recommend that the last major, stable version of the PostgreSQL clients (e.g. 9.6) is installed on the Barman server if you plan to use backup and WAL archiving over streaming replication through `pg_basebackup` and `pg_receivexlog`, for PostgreSQL 9.3 or higher servers. >> **TIP:** For "RPO=0" architectures, it is recommended to have at least one synchronous standby server. barman-2.3/doc/manual/99-references.en.md0000644000076500000240000000534613134472625020717 0ustar mnenciastaff00000000000000 [rpo]: https://en.wikipedia.org/wiki/Recovery_point_objective [rto]: https://en.wikipedia.org/wiki/Recovery_time_objective [repmgr]: http://www.repmgr.org/ [sqldump]: https://www.postgresql.org/docs/current/static/backup-dump.html [recoveryconfig]: https://www.postgresql.org/docs/current/static/recovery-config.html [physicalbackup]: https://www.postgresql.org/docs/current/static/backup-file.html [pitr]: https://www.postgresql.org/docs/current/static/continuous-archiving.html [adminbook]: https://www.2ndquadrant.com/en/books/postgresql-9-administration-cookbook/ [wal]: https://www.postgresql.org/docs/current/static/wal.html [49340627f9821e447f135455d942f7d5e96cae6d]: https://git.postgresql.org/gitweb/?p=postgresql.git;a=commit;h=49340627f9821e447f135455d942f7d5e96cae6d [requirements_recovery]: https://www.postgresql.org/docs/current/static/warm-standby.html#STANDBY-PLANNING [yumpgdg]: http://yum.postgresql.org/ [aptpgdg]: http://apt.postgresql.org/ [aptpgdgwiki]: https://wiki.postgresql.org/wiki/Apt [epel]: http://fedoraproject.org/wiki/EPEL [man5]: http://docs.pgbarman.org/barman.5.html [setup_user]: https://docs.python.org/3/install/index.html#alternate-installation-the-user-scheme [pypi]: https://pypi.python.org/pypi/barman/ [pgpass]: https://www.postgresql.org/docs/current/static/libpq-pgpass.html [pghba]: http://www.postgresql.org/docs/current/static/client-authentication.html [authpghba]: http://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html [streamprot]: http://www.postgresql.org/docs/current/static/protocol-replication.html [roles]: http://www.postgresql.org/docs/current/static/role-attributes.html [replication-slots]: https://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION-SLOTS [synch]: http://www.postgresql.org/docs/current/static/warm-standby.html#SYNCHRONOUS-REPLICATION [3]: https://sourceforge.net/projects/pgbarman/files/ [8]: http://en.wikipedia.org/wiki/Hard_link [9]: https://github.com/2ndquadrant-it/pgespresso [11]: http://www.pgbarman.org/ [12]: http://www.pgbarman.org/support/ [13]: https://www.2ndquadrant.com/ [14]: http://www.pgbarman.org/faq/ [15]: http://blog.2ndquadrant.com/tag/barman/ [16]: https://github.com/hamann/check-barman [17]: https://github.com/2ndquadrant-it/puppet-barman [18]: http://4caast.morfeo-project.org/ [20]: http://www.postgresql.org/docs/current/static/functions-admin.html [24]: http://www.postgresql.org/docs/current/static/warm-standby.html#STREAMING-REPLICATION [25]: http://www.postgresql.org/docs/current/static/app-pgreceivexlog.html [26]: https://goo.gl/218Ghl [27]: https://github.com/emin100/barmanapi [31]: http://www.postgresql.org/ barman-2.3/INSTALL0000644000076500000240000000034113134472625014411 0ustar mnenciastaff00000000000000Barman INSTALL instructions Copyright (C) 2011-2017 2ndQuadrant Limited For further information, see the "Installation" section in the official manual of Barman or the Markdown source file: doc/manual/16-installation.en.md. barman-2.3/LICENSE0000644000076500000240000010451312621123360014360 0ustar mnenciastaff00000000000000 GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU General Public License is a free, copyleft license for software and other kinds of works. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others. For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it. For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions. Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users. Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Use with the GNU Affero General Public License. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode: Copyright (C) This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an "about box". You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see . The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read . barman-2.3/MANIFEST.in0000644000076500000240000000044713066203575015125 0ustar mnenciastaff00000000000000recursive-include barman *.py recursive-include rpm * recursive-include doc *.md recursive-include doc/barman.d * include doc/Makefile doc/barman.1 doc/barman.5 doc/barman.conf include scripts/barman.bash_completion include AUTHORS NEWS ChangeLog LICENSE MANIFEST.in setup.py INSTALL README.rst barman-2.3/NEWS0000644000076500000240000006151013153300354014052 0ustar mnenciastaff00000000000000Barman News - History of user-visible changes Copyright (C) 2011-2017 2ndQuadrant Limited Version 2.3 - 5 Sep 2017 - Add support to PostgreSQL 10 - Follow naming changes in PostgreSQL 10: - The switch-xlog command has been renamed to switch-wal. - In commands output, the xlog word has been changed to WAL and location has been changed to LSN when appropriate. - Add the --network-compression/--no-network-compression options to barman recover to enable or disable network compression at run-time - Add --target-immediate option to recover command, in order to exit recovery when a consistent state is reached (end of the backup, available from PostgreSQL 9.4) - Show cluster state (master or standby) with barman status command - Documentation improvements - Bug fixes: - Fix high memory usage with parallel_jobs > 1 (#116) - Better handling of errors using parallel copy (#114) - Make barman diagnose more robust with system exceptions - Let archive-wal ignore files with .tmp extension Version 2.2 - 17 Jul 2017 - Implement parallel copy for backup/recovery through the parallel_jobs global/server option to be overridden by the --jobs or -j runtime option for the backup and recover command. Parallel backup is available only for the rsync copy method. By default, it is set to 1 (for behaviour compatibility with previous versions). - Support custom WAL size for PostgreSQL 8.4 and newer. At backup time, Barman retrieves from PostgreSQL wal_segment_size and wal_block_size values and computes the necessary calculations. - Improve check command to ensure that incoming directory is empty when archiver=off, and streaming directory is empty when streaming_archiver=off (#80). - Add external_configuration to backup_options so that users can instruct Barman to ignore backup of configuration files when they are not inside PGDATA (default for Debian/Ubuntu installations). In this case, Barman does not display a warning anymore. - Add --get-wal and --no-get-wal options to barman recover - Add max_incoming_wals_queue global/server option for the check command so that a non blocking error is returned in case incoming WAL directories for both archiver and the streaming_archiver contain more files than the specified value. - Documentation improvements - File format changes: - The format of backup.info file has changed. For this reason a backup taken with Barman 2.2 cannot be read by a previous version of Barman. But, backups taken by previous versions can be read by Barman 2.2. - Minor bug fixes: - Allow replication-status to work against a standby - Close any PostgreSQL connection before starting pg_basebackup (#104, #108) - Safely handle paths containing special characters - Archive .partial files after promotion of streaming source - Recursively create directories during recovery (SF#44) - Improve xlog.db locking (#99) - Remove tablespace_map file during recover (#95) - Reconnect to PostgreSQL if connection drops (SF#82) Version 2.1 - 5 Jan 2017 - Add --archive and --archive-timeout options to switch-xlog command - Preliminary support for PostgreSQL 10 (#73) - Minor additions: - Add last archived WAL info to diagnose output - Add start time and execution time to the output of delete command - Minor bug fixes: - Return failure for get-wal command on inactive server - Make streaming_archiver_names and streaming_backup_name options global (#57) - Fix rsync failures due to files truncated during transfer (#64) - Correctly handle compressed history files (#66) - Avoid de-referencing symlinks in pg_tblspc when preparing recovery (#55) - Fix comparison of last archiving failure (#40, #58) - Avoid failing recovery if postgresql.conf is not writable (#68) - Fix output of replication-status command (#56) - Exclude files from backups like pg_basebackup (#65, #72) - Exclude directories from other Postgres versions while copying tablespaces (#74) - Make retry hook script options global Version 2.0 - 27 Sep 2016 - Support for pg_basebackup and base backups over the PostgreSQL streaming replication protocol with backup_method=postgres (PostgreSQL 9.1 or higher required) - Support for physical replication slots through the slot_name configuration option as well as the --create-slot and --drop-slot options for the receive-wal command (PostgreSQL 9.4 or higher required). When slot_name is specified and streaming_archiver is enabled, receive-wal transparently integrates with pg_receivexlog, and check makes sure that slots exist and are actively used - Support for the new backup API introduced in PostgreSQL 9.6, which transparently enables concurrent backups and backups from standby servers using the standard rsync method of backup. Concurrent backup was only possible for PostgreSQL 9.2 to 9.5 versions through the pgespresso extension. The new backup API will make pgespresso redundant in the future - If properly configured, Barman can function as a synchronous standby in terms of WAL streaming. By properly setting the streaming_archiver_name in the synchronous_standby_names priority list on the master, and enabling replication slot support, the receive-wal command can now be part of a PostgreSQL synchronous replication cluster, bringing RPO=0 (PostgreSQL 9.5.5 or higher required) - Introduce barman-wal-restore, a standard and robust script written in Python that can be used as restore_command in recovery.conf files of any standby server of a cluster. It supports remote parallel fetching of WAL files by efficiently invoking get-wal through SSH. Currently available as a separate project called barman-cli. The barman-cli package is required for remote recovery when get-wal is listed in recovery_options - Control the maximum execution time of the check command through the check_timeout global/server configuration option (30 seconds by default) - Limit the number of WAL segments that are processed by an archive-wal run, through the archiver_batch_size and streaming_archiver_batch_size global/server options which control archiving of WAL segments coming from, respectively, the standard archiver and receive-wal - Removed locking of the XLOG database during check operations - The show-backup command is now aware of timelines and properly displays which timelines can be used as recovery targets for a given base backup. Internally, Barman is now capable of parsing .history files - Improved the logic behind the retry mechanism when copy operations experience problems. This involves backup (rsync and postgres) as well as remote recovery (rsync) - Code refactoring involving remote command and physical copy interfaces - Bug fixes: - Correctly handle .history files from streaming - Fix replication-status on PostgreSQL 9.1 - Fix replication-status when sent and write locations are not available - Fix misleading message on pg_receivexlog termination Version 1.6.1 - 23 May 2016 - Add --peek option to get-wal command to discover existing WAL files from the Barman's archive - Add replication-status command for monitoring the status of any streaming replication clients connected to the PostgreSQL server. The --target option allows users to limit the request to only hot standby servers or WAL streaming clients - Add the switch-xlog command to request a switch of a WAL file to the PostgreSQL server. Through the '--force' it issues a CHECKPOINT beforehand - Add streaming_archiver_name option, which sets a proper application_name to pg_receivexlog when streaming_archiver is enabled (only for PostgreSQL 9.3 and above) - Check for _superuser_ privileges with PostgreSQL's standard connections (#30) - Check the WAL archive is never empty - Check for 'backup_label' on the master when server is down - Improve barman-wal-restore contrib script - Bug fixes: - Treat the "failed backups" check as non-fatal - Rename '-x' option for get-wal as '-z' - Add archive_mode=always support for PostgreSQL 9.5 (#32) - Properly close PostgreSQL connections when necessary - Fix receive-wal for pg_receive_xlog version 9.2 Version 1.6.0 - 29 Feb 2016 - Support for streaming replication connection through the streaming_conninfo server option - Support for the streaming_archiver option that allows Barman to receive WAL files through PostgreSQL's native streaming protocol. When set to 'on', it relies on pg_receivexlog to receive WAL data, reducing Recovery Point Objective. Currently, WAL streaming is an additional feature (standard log archiving is still required) - Implement the receive-wal command that, when streaming_archiver is on, wraps pg_receivexlog for WAL streaming. Add --stop option to stop receiving WAL files via streaming protocol. Add --reset option to reset the streaming status and restart from the current xlog in Postgres. - Automatic management (startup and stop) of receive-wal command via cron command - Support for the path_prefix configuration option - Introduction of the archiver option (currently fixed to on) which enables continuous WAL archiving for a specific server, through log shipping via PostgreSQL's archive_command - Support for streaming_wals_directory and errors_directory options - Management of WAL duplicates in archive-wal command and integration with check command - Verify if pg_receivexlog is running in check command when streaming_archiver is enabled - Verify if failed backups are present in check command - Accept compressed WAL files in incoming directory - Add support for the pigz compressor (thanks to Stefano Zacchiroli zack@upsilon.cc) - Implement pygzip and pybzip2 compressors (based on an initial idea of Christoph Moench-Tegeder christoph@2ndquadrant.de) - Creation of an implicit restore point at the end of a backup - Current size of the PostgreSQL data files in barman status - Permit archive_mode=always for PostgreSQL 9.5 servers (thanks to Christoph Moench-Tegeder christoph@2ndquadrant.de) - Complete refactoring of the code responsible for connecting to PostgreSQL - Improve messaging of cron command regarding sub-processes - Native support for Python >= 3.3 - Changes of behaviour: - Stop trashing WAL files during archive-wal (commit:e3a1d16) - Bug fixes: - Atomic WAL file archiving (#9 and #12) - Propagate "-c" option to any Barman subprocess (#19) - Fix management of backup ID during backup deletion (#22) - Improve archive-wal robustness and log messages (#24) - Improve error handling in case of missing parameters Version 1.5.1 - 16 Nov 2015 - Add support for the 'archive-wal' command which performs WAL maintenance operations on a given server - Add support for "per-server" concurrency of the 'cron' command - Improved management of xlog.db errors - Add support for mixed compression types in WAL files (SF.net#61) - Bug fixes: - Avoid retention policy checks during the recovery - Avoid 'wal_level' check on PostgreSQL version < 9.0 (#3) - Fix backup size calculation (#5) Version 1.5.0 - 28 Sep 2015 - Add support for the get-wal command which allows users to fetch any WAL file from the archive of a specific server - Add support for retry hook scripts, a special kind of hook scripts that Barman tries to run until they succeed - Add active configuration option for a server to temporarily disable the server by setting it to False - Add barman_lock_directory global option to change the location of lock files (by default: 'barman_home') - Execute the full suite of checks before starting a backup, and skip it in case one or more checks fail - Forbid to delete a running backup - Analyse include directives of a PostgreSQL server during backup and recover operations - Add check for conflicting paths in the configuration of Barman, both intra (by temporarily disabling a server) and inter-server (by refusing any command, to any server). - Add check for wal_level - Add barman-wal-restore script to be used as restore_command on a standby server, in conjunction with barman get-wal - Implement a standard and consistent policy for error management - Improved cache management of backups - Improved management of configuration in unit tests - Tutorial and man page sources have been converted to Markdown format - Add code documentation through Sphinx - Complete refactor of the code responsible for managing the backup and the recover commands - Changed internal directory structure of a backup - Introduce copy_method option (currently fixed to rsync) - Bug fixes: - Manage options without '=' in PostgreSQL configuration files - Preserve Timeline history files (Fixes: #70) - Workaround for rsync on SUSE Linux (Closes: #13 and #26) - Disables dangerous settings in postgresql.auto.conf (Closes: #68) Version 1.4.1 - 05 May 2015 * Fix for WAL archival stop working if first backup is EMPTY (Closes: #64) * Fix exception during error handling in Barman recovery (Closes: #65) * After a backup, limit cron activity to WAL archiving only (Closes: #62) * Improved robustness and error reporting of the backup delete command (Closes: #63) * Fix computation of WAL production ratio as reported in the show-backup command * Improved management of xlogb file, which is now correctly fsynced when updated. Also, the rebuild-xlogdb command now operates on a temporary new file, which overwrites the main one when finished. * Add unit tests for dateutil module compatibility * Modified Barman version following PEP 440 rules and added support of tests in Python 3.4 Version 1.4.0 - 26 Jan 2015 * Incremental base backup implementation through the reuse_backup global/server option. Possible values are off (disabled, default), copy (preventing unmodified files from being transferred) and link (allowing for deduplication through hard links). * Store and show deduplication effects when using reuse_backup= link. * Added transparent support of pg_stat_archiver (PostgreSQL 9.4) in check, show-server and status commands. * Improved administration by invoking WAL maintenance at the end of a successful backup. * Changed the way unused WAL files are trashed, by differentiating between concurrent and exclusive backup cases. * Improved performance of WAL statistics calculation. * Treat a missing pg_ident.conf as a WARNING rather than an error. * Refactored output layer by removing remaining yield calls. * Check that rsync is in the system path. * Include history files in WAL management. * Improved robustness through more unit tests. * Fixed bug #55: Ignore fsync EINVAL errors on directories. * Fixed bug #58: retention policies delete. Version 1.3.3 - 21 Aug 2014 * Added "last_backup_max_age", a new global/server option that allows administrators to set the max age of the last backup in a catalogue, making it easier to detect any issues with periodical backup execution * Improved robustness of "barman backup" by introducing two global/ server options: "basebackup_retry_times" and "basebackup_retry_sleep". These options allow an administrator to specify, respectively, the number of attempts for a copy operation after a failure, and the number of seconds of wait before retrying * Improved the recovery process via rsync on an existing directory (incremental recovery), by splitting the previous rsync call into several ones - invoking checksum control only when necessary * Added support for PostgreSQL 8.3 * Minor changes: + Support for comma separated list values configuration options + Improved backup durability by calling fsync() on backup and WAL files during "barman backup" and "barman cron" + Improved Nagios output for "barman check --nagios" + Display compression ratio for WALs in "barman show-backup" + Correctly handled keyboard interruption (CTRL-C) while performing barman backup + Improved error messages of failures regarding the stop of a backup + Wider coverage of unit tests * Bug fixes: + Copies "recovery.conf" on the remote server during "barman recover" (#45) + Correctly detect pre/post archive hook scripts (#41) Version 1.3.2 - 15 Apr 2014 * Fixed incompatibility with PostgreSQL 8.4 (Closes #40, bug introduced in version 1.3.1) Version 1.3.1 - 14 Apr 2014 * Added support for concurrent backup of PostgreSQL 9.2 and 9.3 servers that use the "pgespresso" extension. This feature is controlled by the "backup_options" configuration option (global/ server) and activated when set to "concurrent_backup". Concurrent backup allows DBAs to perform full backup operations from a streaming replicated standby. * Added the "barman diagnose" command which prints important information about the Barman system (extremely useful for support and problem solving) * Improved error messages and exception handling interface * Fixed bug in recovery of tablespaces that are created inside the PGDATA directory (bug introduced in version 1.3.0) * Fixed minor bug of unhandled -q option, for quiet mode of commands to be used in cron jobs (bug introduced in version 1.3.0) * Minor bug fixes and code refactoring Version 1.3.0 - 3 Feb 2014 * Refactored BackupInfo class for backup metadata to use the new FieldListFile class (infofile module) * Refactored output layer to use a dedicated module, in order to facilitate integration with Nagios (NagiosOutputWriter class) * Refactored subprocess handling in order to isolate stdin/stderr/ stdout channels (command_wrappers module) * Refactored hook scripts management * Extracted logging configuration and userid enforcement from the configuration class. * Support for hook scripts to be executed before and after a WAL file is archived, through the 'pre_archive_script' and 'post_archive_script' configuration options. * Implemented immediate checkpoint capability with --immediate-checkpoint command option and 'immediate_checkpoint' configuration option * Implemented network compression for remote backup and recovery through the 'network_compression' configuration option (#19) * Implemented the 'rebuild-xlogdb' command (Closes #27 and #28) * Added deduplication of tablespaces located inside the PGDATA directory * Refactored remote recovery code to work the same way local recovery does, by performing remote directory preparation (assuming the remote user has the right permissions on the remote server) * 'barman backup' now tries and create server directories before attempting to execute a full backup (#14) * Fixed bug #22: improved documentation for tablespaces relocation * Fixed bug #31: 'barman cron' checks directory permissions for lock file * Fixed bug #32: xlog.db read access during cron activities Version 1.2.3 - 5 September 2013 * Added support for PostgreSQL 9.3 * Added support for the "--target-name" recovery option, which allows to restore to a named point previously specified with pg_create_restore_point (only for PostgreSQL 9.1 and above users) * Fixed bug #27 about flock() usage with barman.lockfile (many thanks to Damon Snyder ) * Introduced Python 3 compatibility Version 1.2.2 - 24 June 2013 * Fix python 2.6 compatibility Version 1.2.1 - 17 June 2013 * Added the "bandwidth_limit" global/server option which allows to limit the I/O bandwidth (in KBPS) for backup and recovery operations * Added the "tablespace_bandwidth_limit" global/server option which allows to limit the I/O bandwidth (in KBPS) for backup and recovery operations on a per tablespace basis * Added /etc/barman/barman.conf as default location * Bug fix: avoid triggering the minimum_redundancy check on FAILED backups (thanks to Jérôme Vanandruel) Version 1.2.0 - 31 Jan 2013 * Added the "retention_policy_mode" global/server option which defines the method for enforcing retention policies (currently only "auto") * Added the "minimum_redundancy" global/server option which defines the minimum number of backups to be kept for a server * Added the "retention_policy" global/server option which defines retention policies management based on redunancy (e.g. REDUNDANCY 4) or recovery window (e.g. RECOVERY WINDOW OF 3 MONTHS) * Added retention policy support to the logging infrastructure, the "check" and the "status" commands * The "check" command now integrates minimum redundancy control * Added retention policy states (valid, obsolete and potentially obsolete) to "show-backup" and "list-backup" commands * The 'all' keyword is now forbidden as server name * Added basic support for Nagios plugin output to the 'check' command through the --nagios option * Barman now requires argh => 0.21.2 and argcomplete- * Minor bug fixes Version 1.1.2 - 29 Nov 2012 * Added "configuration_files_directory" option that allows to include multiple server configuration files from a directory * Support for special backup IDs: latest, last, oldest, first * Management of multiple servers to the 'list-backup' command. 'barman list-backup all' now list backups for all the configured servers. * Added "application_name" management for PostgreSQL >= 9.0 * Fixed bug #18: ignore missing WAL files if not found during delete Version 1.1.1 - 16 Oct 2012 * Fix regressions in recover command. Version 1.1.0 - 12 Oct 2012 * Support for hook scripts to be executed before and after a 'backup' command through the 'pre_backup_script' and 'post_backup_script' configuration options. * Management of multiple servers to the 'backup' command. 'barman backup all' now iteratively backs up all the configured servers. * Fixed bug #9: "9.2 issue with pg_tablespace_location()" * Add warning in recovery when file location options have been defined in the postgresql.conf file (issue #10) * Fail fast on recover command if the destination directory contains the ':' character (Closes: #4) or if an invalid tablespace relocation rule is passed * Report an informative message when pg_start_backup() invocation fails because an exclusive backup is already running (Closes: #8) Version 1.0.0 - 6 July 2012 * Backup of multiple PostgreSQL servers, with different versions. Versions from PostgreSQL 8.4+ are supported. * Support for secure remote backup (through SSH) * Management of a catalog of backups for every server, allowing users to easily create new backups, delete old ones or restore them * Compression of WAL files that can be configured on a per server basis using compression/decompression filters, both predefined (gzip and bzip2) or custom * Support for INI configuration file with global and per-server directives. Default location for configuration files are /etc/barman.conf or ~/.barman.conf. The '-c' option allows users to specify a different one * Simple indexing of base backups and WAL segments that does not require a local database * Maintenance mode (invoked through the 'cron' command) which performs ordinary operations such as WAL archival and compression, catalog updates, etc. * Added the 'backup' command which takes a full physical base backup of the given PostgreSQL server configured in Barman * Added the 'recover' command which performs local recovery of a given backup, allowing DBAs to specify a point in time. The 'recover' command supports relocation of both the PGDATA directory and, where applicable, the tablespaces * Added the '--remote-ssh-command' option to the 'recover' command for remote recovery of a backup. Remote recovery does not currently support relocation of tablespaces * Added the 'list-server' command that lists all the active servers that have been configured in barman * Added the 'show-server' command that shows the relevant information for a given server, including all configuration options * Added the 'status' command which shows information about the current state of a server, including Postgres version, current transaction ID, archive command, etc. * Added the 'check' command which returns 0 if everything Barman needs is functioning correctly * Added the 'list-backup' command that lists all the available backups for a given server, including size of the base backup and total size of the related WAL segments * Added the 'show-backup' command that shows the relevant information for a given backup, including time of start, size, number of related WAL segments and their size, etc. * Added the 'delete' command which removes a backup from the catalog * Added the 'list-files' command which lists all the files for a single backup * RPM Package for RHEL 5/6 barman-2.3/PKG-INFO0000644000076500000240000000256413153300776014464 0ustar mnenciastaff00000000000000Metadata-Version: 1.1 Name: barman Version: 2.3 Summary: Backup and Recovery Manager for PostgreSQL Home-page: http://www.pgbarman.org/ Author: 2ndQuadrant Limited Author-email: info@2ndquadrant.com License: GPL-3.0 Description: Barman (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. Barman is distributed under GNU GPL 3 and maintained by 2ndQuadrant. Platform: Linux Platform: Mac OS X Classifier: Environment :: Console Classifier: Development Status :: 5 - Production/Stable Classifier: Topic :: System :: Archiving :: Backup Classifier: Topic :: Database Classifier: Topic :: System :: Recovery Tools Classifier: Intended Audience :: System Administrators Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+) Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3.3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 barman-2.3/README.rst0000644000076500000240000000421713134472625015055 0ustar mnenciastaff00000000000000Barman, Backup and Recovery Manager for PostgreSQL ================================================== Barman (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. Barman is distributed under GNU GPL 3 and maintained by 2ndQuadrant. For further information, look at the "Web resources" section below. Source content -------------- Here you can find a description of files and directory distributed with Barman: - AUTHORS : development team of Barman - NEWS : release notes - ChangeLog : log of changes - LICENSE : GNU GPL3 details - TODO : our wishlist for Barman - barman : sources in Python - doc : tutorial and man pages - rpm : SPEC files for RHEL distributions - scripts : auxiliary scripts - tests : unit tests Web resources ------------- - Website : http://www.pgbarman.org/ - Download : http://sourceforge.net/projects/pgbarman/files/ - Documentation : http://www.pgbarman.org/documentation/ - Man page, section 1 : http://docs.pgbarman.org/barman.1.html - Man page, section 5 : http://docs.pgbarman.org/barman.5.html - Community support : http://www.pgbarman.org/support/ - Professional support : https://www.2ndquadrant.com/ - Client utilities : https://github.com/2ndquadrant-it/barman-cli - pgespresso extension : https://github.com/2ndquadrant-it/pgespresso Licence ------- Copyright (C) 2011-2017 2ndQuadrant Limited Barman is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Barman is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Barman. If not, see http://www.gnu.org/licenses/. barman-2.3/rpm/0000755000076500000240000000000013153300776014156 5ustar mnenciastaff00000000000000barman-2.3/rpm/barman.spec0000644000076500000240000001105113153300354016260 0ustar mnenciastaff00000000000000%if 0%{?rhel} == 7 %global pybasever 2.7 %else %if 0%{?fedora}>=21 %global pybasever 2.7 %else %global pybasever 2.6 %endif %endif %if 0%{?rhel} == 5 %global with_python26 1 %endif %if 0%{?with_python26} %global __python_ver python26 %global __python %{_bindir}/python%{pybasever} %global __os_install_post %{__multiple_python_os_install_post} %else %global __python_ver python %endif %global main_version 2.3 # comment out the next line if not a pre-release (use '#%%global ...') #%%global extra_version a1 # Usually 1 - unique sequence for all pre-release version %global package_release 1 %{!?pybasever: %define pybasever %(%{__python} -c "import sys;print(sys.version[0:3])")} %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: Backup and Recovery Manager for PostgreSQL Name: barman Version: %{main_version} Release: %{?extra_version:0.}%{package_release}%{?extra_version:.%{extra_version}}%{?dist} License: GPLv3 Group: Applications/Databases Url: http://www.pgbarman.org/ Source0: %{name}-%{version}%{?extra_version:%{extra_version}}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) BuildArch: noarch Vendor: 2ndQuadrant Limited Requires: python-abi = %{pybasever}, %{__python_ver}-psycopg2 >= 2.4.2, %{__python_ver}-argh >= 0.21.2, %{__python_ver}-argcomplete, %{__python_ver}-dateutil Requires: /usr/sbin/useradd Requires: rsync >= 3.0.4 %description Barman (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. Barman is distributed under GNU GPL 3 and maintained by 2ndQuadrant. %prep %setup -n barman-%{version}%{?extra_version:%{extra_version}} -q %build %{__python} setup.py build cat > barman.cron << EOF # m h dom mon dow user command * * * * * barman [ -x %{_bindir}/barman ] && %{_bindir}/barman -q cron EOF cat > barman.logrotate << EOF /var/log/barman/barman.log { missingok notifempty create 0600 barman barman } EOF %install %{__python} setup.py install -O1 --skip-build --root %{buildroot} mkdir -p %{buildroot}%{_sysconfdir}/bash_completion.d mkdir -p %{buildroot}%{_sysconfdir}/cron.d/ mkdir -p %{buildroot}%{_sysconfdir}/logrotate.d/ mkdir -p %{buildroot}%{_sysconfdir}/barman.d/ mkdir -p %{buildroot}/var/lib/barman mkdir -p %{buildroot}/var/log/barman install -pm 644 doc/barman.conf %{buildroot}%{_sysconfdir}/barman.conf install -pm 644 doc/barman.d/* %{buildroot}%{_sysconfdir}/barman.d/ install -pm 644 scripts/barman.bash_completion %{buildroot}%{_sysconfdir}/bash_completion.d/barman install -pm 644 barman.cron %{buildroot}%{_sysconfdir}/cron.d/barman install -pm 644 barman.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/barman touch %{buildroot}/var/log/barman/barman.log %clean rm -rf %{buildroot} %files %defattr(-,root,root) %doc INSTALL NEWS README.rst %{python_sitelib}/%{name}-%{version}%{?extra_version:%{extra_version}}-py%{pybasever}.egg-info %{python_sitelib}/%{name}/ %{_bindir}/%{name} %doc %{_mandir}/man1/%{name}.1.gz %doc %{_mandir}/man5/%{name}.5.gz %config(noreplace) %{_sysconfdir}/bash_completion.d/ %config(noreplace) %{_sysconfdir}/%{name}.conf %config(noreplace) %{_sysconfdir}/cron.d/%{name} %config(noreplace) %{_sysconfdir}/logrotate.d/%{name} %config(noreplace) %{_sysconfdir}/barman.d/ %attr(700,barman,barman) %dir /var/lib/%{name} %attr(755,barman,barman) %dir /var/log/%{name} %attr(600,barman,barman) %ghost /var/log/%{name}/%{name}.log %pre groupadd -f -r barman >/dev/null 2>&1 || : useradd -M -n -g barman -r -d /var/lib/barman -s /bin/bash \ -c "Backup and Recovery Manager for PostgreSQL" barman >/dev/null 2>&1 || : %changelog * Tue Sep 5 2017 - Marco Nenciarini 2.3-1 - New release 2.3-1 * Mon Jul 17 2017 - Marco Nenciarini 2.2-1 - New release 2.2-1 * Thu Jan 5 2017 - Giulio Calacoci 2.1-1 - New release 2.1-1 * Tue Dec 27 2016 - Gabriele Bartolini 2.1-0.1.alpha.1 - New release 2.1-0.1.alpha.1 * Tue Sep 27 2016 - Gabriele Bartolini 2.0-1 - New release 2.0-1 - Trim changelog for releases 1.X barman-2.3/rpm/rhel5/0000755000076500000240000000000013153300776015175 5ustar mnenciastaff00000000000000barman-2.3/rpm/rhel5/python-dateutil-1.4.1-remove-embedded-timezone-data.patch0000644000076500000240000000636712404074014027573 0ustar mnenciastaff00000000000000diff -up python-dateutil-1.4.1/dateutil/tz.py.remove-embedded-timezone-data python-dateutil-1.4.1/dateutil/tz.py --- python-dateutil-1.4.1/dateutil/tz.py.remove-embedded-timezone-data 2008-02-27 20:45:41.000000000 -0500 +++ python-dateutil-1.4.1/dateutil/tz.py 2010-07-13 14:40:30.228122861 -0400 @@ -930,9 +930,6 @@ def gettz(name=None): except OSError: pass if not tz: - from dateutil.zoneinfo import gettz - tz = gettz(name) - if not tz: for c in name: # name must have at least one offset to be a tzstr if c in "0123456789": diff -up python-dateutil-1.4.1/dateutil/zoneinfo/__init__.py.remove-embedded-timezone-data python-dateutil-1.4.1/dateutil/zoneinfo/__init__.py --- python-dateutil-1.4.1/dateutil/zoneinfo/__init__.py.remove-embedded-timezone-data 2005-12-22 13:13:50.000000000 -0500 +++ python-dateutil-1.4.1/dateutil/zoneinfo/__init__.py 2010-07-13 14:40:30.228122861 -0400 @@ -3,6 +3,10 @@ Copyright (c) 2003-2005 Gustavo Niemeye This module offers extensions to the standard python 2.3+ datetime module. + +This version of the code has been modified to remove the embedded copy +of zoneinfo-2008e.tar.gz and instead use the system data from the tzdata +package """ from dateutil.tz import tzfile from tarfile import TarFile @@ -13,49 +17,12 @@ __license__ = "PSF License" __all__ = ["setcachesize", "gettz", "rebuild"] -CACHE = [] -CACHESIZE = 10 - -class tzfile(tzfile): - def __reduce__(self): - return (gettz, (self._filename,)) - -def getzoneinfofile(): - filenames = os.listdir(os.path.join(os.path.dirname(__file__))) - filenames.sort() - filenames.reverse() - for entry in filenames: - if entry.startswith("zoneinfo") and ".tar." in entry: - return os.path.join(os.path.dirname(__file__), entry) - return None - -ZONEINFOFILE = getzoneinfofile() - -del getzoneinfofile - def setcachesize(size): - global CACHESIZE, CACHE - CACHESIZE = size - del CACHE[size:] + pass def gettz(name): - tzinfo = None - if ZONEINFOFILE: - for cachedname, tzinfo in CACHE: - if cachedname == name: - break - else: - tf = TarFile.open(ZONEINFOFILE) - try: - zonefile = tf.extractfile(name) - except KeyError: - tzinfo = None - else: - tzinfo = tzfile(zonefile) - tf.close() - CACHE.insert(0, (name, tzinfo)) - del CACHE[CACHESIZE:] - return tzinfo + from dateutil.tz import gettz + return gettz(name) def rebuild(filename, tag=None, format="gz"): import tempfile, shutil diff -up python-dateutil-1.4.1/MANIFEST.in.remove-embedded-timezone-data python-dateutil-1.4.1/MANIFEST.in --- python-dateutil-1.4.1/MANIFEST.in.remove-embedded-timezone-data 2010-07-13 14:42:07.974118722 -0400 +++ python-dateutil-1.4.1/MANIFEST.in 2010-07-13 14:42:14.409994960 -0400 @@ -1,4 +1,4 @@ -recursive-include dateutil *.py *.tar.* +recursive-include dateutil *.py recursive-include sandbox *.py include setup.py setup.cfg MANIFEST.in README LICENSE NEWS Makefile include test.py example.py barman-2.3/rpm/rhel5/python26-argcomplete.spec0000644000076500000240000000417012541044263022040 0ustar mnenciastaff00000000000000# Use Python 2.6 %global pybasever 2.6 %global __python_ver 26 %global __python %{_bindir}/python%{pybasever} %global __os_install_post %{__multiple_python_os_install_post} %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: Bash tab completion for argparse Name: python%{__python_ver}-argcomplete Version: 0.3.5 Release: 1%{?dist} License: ASL 2.0 Group: Development/Libraries Url: https://github.com/kislyuk/argcomplete Source0: http://pypi.python.org/packages/source/a/argcomplete/argcomplete-%{version}.tar.gz BuildRequires: python%{__python_ver}-devel,python%{__python_ver}-setuptools BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) BuildArch: noarch Requires: python-abi = %(%{__python} -c "import sys ; print sys.version[:3]") %if "%{__python_ver}" == "26" Requires: python%{__python_ver}-argparse %endif %description Argcomplete provides easy, extensible command line tab completion of arguments for your Python script. It makes two assumptions: * You're using bash as your shell * You're using argparse to manage your command line arguments/options Argcomplete is particularly useful if your program has lots of options or subparsers, and if your program can dynamically suggest completions for your argument/option values (for example, if the user is browsing resources over the network). %prep %setup -n argcomplete-%{version} -q %build %{__python} setup.py build %install %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT %clean rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) %doc README.rst %{python_sitelib}/argcomplete-%{version}-py%{pybasever}.egg-info %{python_sitelib}/argcomplete/ %{_bindir}/activate-global-python-argcomplete %{_bindir}/python-argcomplete-check-easy-install-script %{_bindir}/register-python-argcomplete %changelog * Thu Jan 31 2013 - Marco Neciarini 0.3.5-1 - Initial packaging. barman-2.3/rpm/rhel5/python26-argh.spec0000644000076500000240000000376212541044263020465 0ustar mnenciastaff00000000000000# Use Python 2.6 %global pybasever 2.6 %global __python_ver 26 %global __python %{_bindir}/python%{pybasever} %global __os_install_post %{__multiple_python_os_install_post} %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: A simple argparse wrapper Name: python%{__python_ver}-argh Version: 0.23.0 Release: 1%{?dist} License: LGPLv3 Group: Development/Libraries Url: http://bitbucket.org/neithere/argh/ Source0: http://pypi.python.org/packages/source/a/argh/argh-%{version}.tar.gz BuildRequires: python%{__python_ver}-devel,python%{__python_ver}-setuptools BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) BuildArch: noarch Requires: python-abi = %(%{__python} -c "import sys ; print sys.version[:3]") %if "%{__python_ver}" == "26" Requires: python%{__python_ver}-argparse %endif %description Agrh, argparse! =============== Did you ever say "argh" trying to remember the details of optparse or argparse API? If yes, this package may be useful for you. It provides a very simple wrapper for argparse with support for hierarchical commands that can be bound to modules or classes. Argparse can do it; argh makes it easy. %prep %setup -n argh-%{version} -q %build %{__python} setup.py build %install %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT %clean rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) %doc README %{python_sitelib}/argh-%{version}-py%{pybasever}.egg-info %{python_sitelib}/argh/ %changelog * Thu Jan 31 2013 - Marco Neciarini 0.23.0-1 - Update to version 0.23.0 * Wed May 9 2012 - Marco Neciarini 0.15.0-1 - Update to version 0.15.0 * Sat Dec 4 2011 - Marco Neciarini 0.14.2-1 - Initial packaging. barman-2.3/rpm/rhel5/python26-dateutil.spec0000644000076500000240000001014412404074014021342 0ustar mnenciastaff00000000000000# Use Python 2.6 %global pybasever 2.6 %global __python_ver 26 %global __python %{_bindir}/python%{pybasever} %global __os_install_post %{__multiple_python_os_install_post} %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Name: python%{__python_ver}-dateutil Version: 1.4.1 Release: 6%{?dist} Summary: Powerful extensions to the standard datetime module Group: Development/Languages License: Python URL: http://labix.org/python-dateutil Source0: http://labix.org/download/python-dateutil/python-dateutil-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) # Redirect the exposed parts of the dateutil.zoneinfo API to remove references # to the embedded copy of zoneinfo-2008e.tar.gz and instead use the system # data from the "tzdata" package (rhbz#559309): Patch0: python-dateutil-1.4.1-remove-embedded-timezone-data.patch BuildArch: noarch BuildRequires: python%{__python_ver}-devel,python%{__python_ver}-setuptools Requires: tzdata %description The dateutil module provides powerful extensions to the standard datetime module available in Python 2.3+. %prep %setup -n python-dateutil-%{version} -q # Remove embedded copy of timezone data: %patch0 -p1 rm dateutil/zoneinfo/zoneinfo-2008e.tar.gz # Change encoding of NEWS file to UTF-8, preserving timestamp: iconv -f ISO-8859-1 -t utf8 NEWS > NEWS.utf8 && \ touch -r NEWS NEWS.utf8 && \ mv NEWS.utf8 NEWS %build %{__python} setup.py build %install rm -rf $RPM_BUILD_ROOT %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT %clean rm -rf $RPM_BUILD_ROOT %check %{__python} test.py %files %defattr(-,root,root,-) %doc example.py LICENSE NEWS README %{python_sitelib}/dateutil/ %{python_sitelib}/*.egg-info %changelog * Tue Jul 13 2010 David Malcolm - 1.4.1-6 - remove embedded copy of timezone data, and redirect the dateutil.zoneinfo API accordingly Resolves: rhbz#559309 - add a %%check, running the upstream selftest suite * Tue Jul 13 2010 David Malcolm - 1.4.1-5 - add requirement on tzdata Resolves: rhbz#559309 - fix encoding of the NEWS file * Mon Nov 30 2009 Dennis Gregorovic - 1.4.1-4.1 - Rebuilt for RHEL 6 * Sun Jul 26 2009 Fedora Release Engineering - 1.4.1-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_12_Mass_Rebuild * Thu Feb 26 2009 Fedora Release Engineering - 1.4.1-3 - Rebuilt for https://fedoraproject.org/wiki/Fedora_11_Mass_Rebuild * Fri Feb 20 2009 Jef Spaleta - 1.4.1-2 - small specfile fix * Fri Feb 20 2009 Jef Spaleta - 1.4.1-2 - New upstream version * Sat Nov 29 2008 Ignacio Vazquez-Abrams - 1.4-3 - Rebuild for Python 2.6 * Fri Aug 29 2008 Tom "spot" Callaway - 1.4-2 - fix license tag * Tue Jul 01 2008 Jef Spaleta 1.4-1 - Latest upstream release * Fri Jan 04 2008 Jef Spaleta 1.2-2 - Fix for egg-info file creation * Thu Jun 28 2007 Orion Poplawski 1.2-1 - Update to 1.2 * Mon Dec 11 2006 Jef Spaleta 1.1-5 - Fix python-devel BR, as per discussion in maintainers-list * Mon Dec 11 2006 Jef Spaleta 1.1-4 - Release bump for rebuild against python 2.5 in devel tree * Wed Jul 26 2006 Orion Poplawski 1.1-3 - Add patch to fix building on x86_64 * Wed Feb 15 2006 Orion Poplawski 1.1-2 - Rebuild for gcc/glibc changes * Thu Dec 22 2005 Orion Poplawski 1.1-1 - Update to 1.1 * Thu Jul 28 2005 Orion Poplawski 1.0-1 - Update to 1.0 * Tue Jul 05 2005 Orion Poplawski 0.9-1 - Initial Fedora Extras package barman-2.3/rpm/rhel5/python26-psycopg2.spec0000644000076500000240000001400012404074014021270 0ustar mnenciastaff00000000000000# Use Python 2.6 %global pybasever 2.6 %global __python_ver 26 %global __python %{_bindir}/python%{pybasever} %global __os_install_post %{__multiple_python_os_install_post} %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} %define ZPsycopgDAdir %{_localstatedir}/lib/zope/Products/ZPsycopgDA %global pgmajorversion 90 %global pginstdir /usr/pgsql-9.0 %global sname psycopg2 Summary: A PostgreSQL database adapter for Python Name: python26-%{sname} Version: 2.4.5 Release: 1%{?dist} License: LGPLv3 with exceptions Group: Applications/Databases Url: http://www.psycopg.org/psycopg/ Source0: http://initd.org/psycopg/tarballs/PSYCOPG-2-4/%{sname}-%{version}.tar.gz Patch0: setup.cfg.patch BuildRequires: python%{__python_ver}-devel postgresql%{pgmajorversion}-devel BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) Requires: python-abi = %(%{__python} -c "import sys ; print sys.version[:3]") %description psycopg is a PostgreSQL database adapter for the Python programming language (just like pygresql and popy.) It was written from scratch with the aim of being very small and fast, and stable as a rock. The main advantages of psycopg are that it supports the full Python DBAPI-2.0 and being thread safe at level 2. %package doc Summary: Documentation for psycopg python PostgreSQL database adapter Group: Documentation Requires: %{name} = %{version}-%{release} %description doc Documentation and example files for the psycopg python PostgreSQL database adapter. %package test Summary: Tests for psycopg2 Group: Development/Libraries Requires: %{name} = %{version}-%{release} %description test Tests for psycopg2. %package zope Summary: Zope Database Adapter ZPsycopgDA Group: Applications/Databases Requires: %{name} = %{version}-%{release} zope %description zope Zope Database Adapter for PostgreSQL, called ZPsycopgDA %prep %setup -q -n psycopg2-%{version} %patch0 -p0 %build %{__python} setup.py build # Fix for wrong-file-end-of-line-encoding problem; upstream also must fix this. for i in `find doc -iname "*.html"`; do sed -i 's/\r//' $i; done for i in `find doc -iname "*.css"`; do sed -i 's/\r//' $i; done %install rm -Rf %{buildroot} mkdir -p %{buildroot}%{python_sitearch}/psycopg2 %{__python} setup.py install --no-compile --root %{buildroot} install -d %{buildroot}%{ZPsycopgDAdir} cp -pr ZPsycopgDA/* %{buildroot}%{ZPsycopgDAdir} %clean rm -rf %{buildroot} %files %defattr(-,root,root) %doc AUTHORS ChangeLog INSTALL LICENSE README %dir %{python_sitearch}/psycopg2 %{python_sitearch}/psycopg2/*.py %{python_sitearch}/psycopg2/*.pyc %{python_sitearch}/psycopg2/*.so %{python_sitearch}/psycopg2/*.pyo %{python_sitearch}/psycopg2-*.egg-info %files doc %defattr(-,root,root) %doc doc examples/ %files test %defattr(-,root,root) %{python_sitearch}/%{sname}/tests/* %files zope %defattr(-,root,root) %dir %{ZPsycopgDAdir} %{ZPsycopgDAdir}/*.py %{ZPsycopgDAdir}/*.pyo %{ZPsycopgDAdir}/*.pyc %{ZPsycopgDAdir}/dtml/* %{ZPsycopgDAdir}/icons/* %changelog * Wed May 9 2012 - Marco Neciarini 2.4.5-1 - Update to version 2.4.5 * Mon Aug 22 2011 Devrim GUNDUZ 2.4.2-1 - Update to 2.4.2 - Add a patch for pg_config path. - Add new subpackage: test * Tue Mar 16 2010 Devrim GUNDUZ 2.0.14-1 - Update to 2.0.14 * Mon Oct 19 2009 Devrim GUNDUZ 2.0.13-1 - Update to 2.0.13 * Mon Sep 7 2009 Devrim GUNDUZ 2.0.12-1 - Update to 2.0.12 * Tue May 26 2009 Devrim GUNDUZ 2.0.11-1 - Update to 2.0.11 * Fri Apr 24 2009 Devrim GUNDUZ 2.0.10-1 - Update to 2.0.10 * Thu Mar 2 2009 Devrim GUNDUZ 2.0.9-1 - Update to 2.0.9 * Wed Apr 30 2008 - Devrim GUNDUZ 2.0.7-1 - Update to 2.0.7 * Fri Jun 15 2007 - Devrim GUNDUZ 2.0.6-1 - Update to 2.0.6 * Sun May 06 2007 Thorsten Leemhuis - rebuilt for RHEL5 final * Wed Dec 6 2006 - Devrim GUNDUZ 2.0.5.1-4 - Rebuilt for PostgreSQL 8.2.0 * Mon Sep 11 2006 - Devrim GUNDUZ 2.0.5.1-3 - Rebuilt * Wed Sep 6 2006 - Devrim GUNDUZ 2.0.5.1-2 - Remove ghost'ing, per Python Packaging Guidelines * Mon Sep 4 2006 - Devrim GUNDUZ 2.0.5.1-1 - Update to 2.0.5.1 * Sun Aug 6 2006 - Devrim GUNDUZ 2.0.3-3 - Fixed zope package dependencies and macro definition, per bugzilla review (#199784) - Fixed zope package directory ownership, per bugzilla review (#199784) - Fixed cp usage for zope subpackage, per bugzilla review (#199784) * Mon Jul 31 2006 - Devrim GUNDUZ 2.0.3-2 - Fixed 64 bit builds - Fixed license - Added Zope subpackage - Fixed typo in doc description - Added macro for zope subpackage dir * Mon Jul 31 2006 - Devrim GUNDUZ 2.0.3-1 - Update to 2.0.3 - Fixed spec file, per bugzilla review (#199784) * Sat Jul 22 2006 - Devrim GUNDUZ 2.0.2-3 - Removed python dependency, per bugzilla review. (#199784) - Changed doc package group, per bugzilla review. (#199784) - Replaced dos2unix with sed, per guidelines and bugzilla review (#199784) - Fix changelog dates * Sat Jul 21 2006 - Devrim GUNDUZ 2.0.2-2 - Added dos2unix to buildrequires - removed python related part from package name * Fri Jul 20 2006 - Devrim GUNDUZ 2.0.2-1 - Fix rpmlint errors, including dos2unix solution - Re-engineered spec file * Fri Jan 23 2006 - Devrim GUNDUZ - First 2.0.X build * Fri Jan 23 2006 - Devrim GUNDUZ - Update to 1.2.21 * Tue Dec 06 2005 - Devrim GUNDUZ - Initial release for 1.1.20 barman-2.3/rpm/rhel5/setup.cfg.patch0000644000076500000240000000100112404074014020073 0ustar mnenciastaff00000000000000--- setup.cfg.old 2011-08-22 12:16:18.703486005 +0300 +++ setup.cfg 2011-08-22 12:16:31.596486005 +0300 @@ -26,7 +26,7 @@ # libraries needed to build psycopg2. If pg_config is not in the path or # is installed under a different name uncomment the following option and # set it to the pg_config full path. -#pg_config= +pg_config=/usr/pgsql-9.0/bin/pg_config # If "pg_config" is not available, "include_dirs" can be used to locate # postgresql headers and libraries. Some extra checks on sys.platform will barman-2.3/rpm/rhel6/0000755000076500000240000000000013153300776015176 5ustar mnenciastaff00000000000000barman-2.3/rpm/rhel6/python-argcomplete.spec0000644000076500000240000000353712541044263021677 0ustar mnenciastaff00000000000000%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: Bash tab completion for argparse Name: python-argcomplete Version: 0.3.5 Release: 1%{?dist} License: ASL 2.0 Group: Development/Libraries Url: https://github.com/kislyuk/argcomplete Source0: http://pypi.python.org/packages/source/a/argcomplete/argcomplete-%{version}.tar.gz BuildRequires: python-devel,python-setuptools BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) BuildArch: noarch Requires: python-abi = %(%{__python} -c "import sys ; print sys.version[:3]") Requires: python-argparse %description Argcomplete provides easy, extensible command line tab completion of arguments for your Python script. It makes two assumptions: * You're using bash as your shell * You're using argparse to manage your command line arguments/options Argcomplete is particularly useful if your program has lots of options or subparsers, and if your program can dynamically suggest completions for your argument/option values (for example, if the user is browsing resources over the network). %prep %setup -n argcomplete-%{version} -q %build %{__python} setup.py build %install %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT %clean rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) %doc README.rst %{python_sitelib}/argcomplete-%{version}-py2.6.egg-info %{python_sitelib}/argcomplete/ %{_bindir}/activate-global-python-argcomplete %{_bindir}/python-argcomplete-check-easy-install-script %{_bindir}/register-python-argcomplete %changelog * Thu Jan 31 2013 - Marco Neciarini 0.3.5-1 - Initial packaging. barman-2.3/rpm/rhel6/python-argh.spec0000644000076500000240000000333212627040732020311 0ustar mnenciastaff00000000000000%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: A simple argparse wrapper Name: python-argh Version: 0.23.0 Release: 1%{?dist} License: LGPLv3 Group: Development/Libraries Url: http://bitbucket.org/neithere/argh/ Source0: http://pypi.python.org/packages/source/a/argh/argh-%{version}.tar.gz BuildRequires: python-devel, python-setuptools BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) BuildArch: noarch Requires: python-abi = %(%{__python} -c "import sys ; print sys.version[:3]") Requires: python-argparse %description Argh, argparse! =============== Did you ever say "argh" trying to remember the details of optparse or argparse API? If yes, this package may be useful for you. It provides a very simple wrapper for argparse with support for hierarchical commands that can be bound to modules or classes. Argparse can do it; argh makes it easy. %prep %setup -n argh-%{version} -q %build %{__python} setup.py build %install %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT %clean rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) %doc README %{python_sitelib}/argh-%{version}-py2.6.egg-info %{python_sitelib}/argh/ %changelog * Thu Jan 31 2013 - Marco Neciarini 0.23.0-1 - Update to version 0.23.0 * Wed May 9 2012 - Marco Neciarini 0.15.0-1 - Update to version 0.15.0 * Sat Dec 4 2011 - Marco Neciarini 0.14.2-1 - Initial packaging. barman-2.3/rpm/rhel7/0000755000076500000240000000000013153300776015177 5ustar mnenciastaff00000000000000barman-2.3/rpm/rhel7/python-argh.spec0000644000076500000240000000352012627040732020311 0ustar mnenciastaff00000000000000%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} %{!?python_sitearch: %define python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib(1)")} Summary: A simple argparse wrapper Name: python-argh Version: 0.26.1 Release: 1%{?dist} License: LGPLv3 Group: Development/Libraries Url: http://bitbucket.org/neithere/argh/ Source0: http://pypi.python.org/packages/source/a/argh/argh-%{version}.tar.gz BuildRequires: python-devel, python-setuptools BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot-%(%{__id_u} -n) BuildArch: noarch Requires: python-abi = %(%{__python} -c "import sys ; print sys.version[:3]") Requires: python-argparse %description Argh, argparse! =============== Did you ever say "argh" trying to remember the details of optparse or argparse API? If yes, this package may be useful for you. It provides a very simple wrapper for argparse with support for hierarchical commands that can be bound to modules or classes. Argparse can do it; argh makes it easy. %prep %setup -n argh-%{version} -q %build %{__python} setup.py build %install %{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT %clean rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) %doc README.rst %{python_sitelib}/argh-%{version}-py2.7.egg-info %{python_sitelib}/argh/ %changelog * Tue Jan 20 2015 - Francesco Canovai 0.26.1-1 - Update to version 0.26.1 * Thu Jan 31 2013 - Marco Nenciarini 0.23.0-1 - Update to version 0.23.0 * Wed May 9 2012 - Marco Nenciarini 0.15.0-1 - Update to version 0.15.0 * Sat Dec 3 2011 - Marco Nenciarini 0.14.2-1 - Initial packaging. barman-2.3/scripts/0000755000076500000240000000000013153300776015047 5ustar mnenciastaff00000000000000barman-2.3/scripts/barman.bash_completion0000644000076500000240000000005512541044263021373 0ustar mnenciastaff00000000000000eval "$(register-python-argcomplete barman)" barman-2.3/setup.cfg0000644000076500000240000000034413153300776015202 0ustar mnenciastaff00000000000000[bdist_wheel] universal = 1 [aliases] test = pytest [isort] known_first_party = barman known_third_party = setuptools,distutils,argh,argcomplete,dateutil,psycopg2,mock,pytest skip = .tox [egg_info] tag_build = tag_date = 0 barman-2.3/setup.py0000755000076500000240000000645013134475447015111 0ustar mnenciastaff00000000000000#!/usr/bin/env python # # barman - Backup and Recovery Manager for PostgreSQL # # Copyright (C) 2011-2017 2ndQuadrant Limited # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . """Backup and Recovery Manager for PostgreSQL Barman (Backup and Recovery Manager) is an open-source administration tool for disaster recovery of PostgreSQL servers written in Python. It allows your organisation to perform remote backups of multiple servers in business critical environments to reduce risk and help DBAs during the recovery phase. Barman is distributed under GNU GPL 3 and maintained by 2ndQuadrant. """ import sys try: from setuptools import setup except ImportError: from distutils.core import setup if sys.version_info < (2, 6): raise SystemExit('ERROR: Barman needs at least python 2.6 to work') # Depend on pytest_runner only when the tests are actually invoked needs_pytest = set(['pytest', 'test']).intersection(sys.argv) pytest_runner = ['pytest_runner'] if needs_pytest else [] setup_requires = pytest_runner install_requires = [ 'psycopg2 >= 2.4.2', 'argh >= 0.21.2, <= 0.26.2', 'python-dateutil', 'argcomplete', ] if sys.version_info < (2, 7): install_requires += [ 'argparse', ] barman = {} with open('barman/version.py', 'r') as fversion: exec(fversion.read(), barman) setup( name='barman', version=barman['__version__'], author='2ndQuadrant Limited', author_email='info@2ndquadrant.com', url='http://www.pgbarman.org/', packages=['barman', ], scripts=['bin/barman', ], data_files=[ ('share/man/man1', ['doc/barman.1']), ('share/man/man5', ['doc/barman.5']), ], license='GPL-3.0', description=__doc__.split("\n")[0], long_description="\n".join(__doc__.split("\n")[2:]), install_requires=install_requires, platforms=['Linux', 'Mac OS X'], classifiers=[ 'Environment :: Console', 'Development Status :: 5 - Production/Stable', 'Topic :: System :: Archiving :: Backup', 'Topic :: Database', 'Topic :: System :: Recovery Tools', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: GNU General Public License v3 or later ' '(GPLv3+)', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', ], setup_requires=setup_requires, tests_require=[ 'mock', 'pytest-catchlog>=1.2.1', 'pytest-timeout', 'pytest', ], )