mysql-mmm-2.2.1/ 0000755 0000000 0000000 00000000000 11370745632 012153 5 ustar root root mysql-mmm-2.2.1/VERSION 0000644 0000000 0000000 00000000006 11370745632 013217 0 ustar root root 2.2.1
mysql-mmm-2.2.1/INSTALL 0000644 0000000 0000000 00000000127 11370745632 013204 0 ustar root root
1. Install dependencies
See .pdf-Documentation.
2. Install mysql-mmm
make install
mysql-mmm-2.2.1/README 0000644 0000000 0000000 00000000000 11370745632 013021 0 ustar root root mysql-mmm-2.2.1/etc/ 0000755 0000000 0000000 00000000000 11370745632 012726 5 ustar root root mysql-mmm-2.2.1/etc/mysql-mmm/ 0000755 0000000 0000000 00000000000 11370745632 014657 5 ustar root root mysql-mmm-2.2.1/etc/mysql-mmm/mmm_tools.conf 0000640 0000000 0000000 00000002415 11370745632 017532 0 ustar root root include mmm_agent.conf
default_copy_method scp
clone_dirs data, logs
ssh_user root
lvm_snapshot_size 1G
lvm_logical_volume mysql
lvm_volume_group storage
lvm_mount_dir /mmm_snapshot
lvm_mount_opts -orw,nouuid
tools_user mmm_tools
tools_password RepTools
backup_dir /mmm_backup
restore_dir /mysql
backup_command scp -c blowfish -r %SSH_USER%@%IP%:%SNAPSHOT%/%CLONE_DIR% %DEST_DIR%/
restore_command cp -axv %BACKUP_DIR%/* %DEST_DIR%/
true_copy 1
backup_command rdiff-backup --ssh-no-compression -v 5 !--include %SNAPSHOT%/%CLONE_DIR%! --exclude %SNAPSHOT% %SSH_USER%@%IP%::%SNAPSHOT%/ %DEST_DIR%/
restore_command rdiff-backup --force -v 5 -r %VERSION% %BACKUP_DIR% %DEST_DIR%/.mmm_restore; cp -axvl --remove-destination %DEST_DIR%/.mmm_restore/* %DEST_DIR%/; rm -r %DEST_DIR%/.mmm_restore/
incremental_command rdiff-backup --parsable-output -l %BACKUP_DIR%
single_run 1
incremental 1
backup_command ssh -c blowfish %SSH_USER%@%IP% "cd '%SNAPSHOT%'; tar cv !'%CLONE_DIR%'!" | gzip > %DEST_DIR%/backup.tar.gz
restore_command cd %DEST_DIR%; tar xzfv %BACKUP_DIR%/backup.tar.gz
single_run 1
mysql-mmm-2.2.1/etc/mysql-mmm/mmm_mon.conf 0000640 0000000 0000000 00000000501 11370745632 017155 0 ustar root root include mmm_common.conf
ip 127.0.0.1
pid_path /var/run/mmm_mond.pid
bin_path /usr/lib/mysql-mmm/
status_path /var/lib/misc/mmm_mond.status
ping_ips 192.168.0.1, 192.168.0.2, 192.168.0.3
monitor_user mmm_monitor
monitor_password RepMonitor
debug 0
mysql-mmm-2.2.1/etc/mysql-mmm/mmm_agent.conf 0000640 0000000 0000000 00000000041 11370745632 017461 0 ustar root root include mmm_common.conf
this db1
mysql-mmm-2.2.1/etc/mysql-mmm/mmm_common.conf 0000640 0000000 0000000 00000001254 11370745632 017662 0 ustar root root active_master_role writer
cluster_interface eth0
pid_path /var/run/mmm_agentd.pid
bin_path /usr/lib/mysql-mmm/
replication_user replication
replication_password slave
agent_user mmm_agent
agent_password RepAgent
ip 192.168.0.31
mode master
peer db2
ip 192.168.0.32
mode master
peer db1
ip 192.168.0.33
mode slave
hosts db1, db2
ips 192.168.0.50
mode exclusive
hosts db1, db2, db3
ips 192.168.0.51, 192.168.0.52, 192.168.0.53
mode balanced
mysql-mmm-2.2.1/etc/init.d/ 0000755 0000000 0000000 00000000000 11370745632 014113 5 ustar root root mysql-mmm-2.2.1/etc/init.d/mysql-mmm-agent 0000755 0000000 0000000 00000004510 11370745632 017066 0 ustar root root #!/bin/sh
#
# mysql-mmm-agent This shell script takes care of starting and stopping
# the mmm agent daemon.
#
# chkconfig: - 64 36
# description: MMM Agent.
# processname: mmm_agentd
# config: /etc/mmm_agent.conf
# pidfile: /var/run/mmm_agentd.pid
# Cluster name (it can be empty for default cases)
CLUSTER=''
#-----------------------------------------------------------------------
# Paths
if [ "$CLUSTER" != "" ]; then
MMM_AGENTD_BIN="/usr/sbin/mmm_agentd @$CLUSTER"
MMM_AGENTD_PIDFILE="/var/run/mmm_agentd-$CLUSTER.pid"
else
MMM_AGENTD_BIN="/usr/sbin/mmm_agentd"
MMM_AGENTD_PIDFILE="/var/run/mmm_agentd.pid"
fi
echo "Daemon bin: '$MMM_AGENTD_BIN'"
echo "Daemon pid: '$MMM_AGENTD_PIDFILE'"
#-----------------------------------------------------------------------
# See how we were called.
case "$1" in
start)
# Start daemon.
echo -n "Starting MMM Agent daemon... "
if [ -s $MMM_AGENTD_PIDFILE ] && kill -0 `cat $MMM_AGENTD_PIDFILE` 2> /dev/null; then
echo " already running."
exit 0
fi
$MMM_AGENTD_BIN
if [ "$?" -ne 0 ]; then
echo "failed"
exit 1
fi
echo "Ok"
exit 0
;;
stop)
# Stop daemon.
echo -n "Shutting down MMM Agent daemon"
if [ -s $MMM_AGENTD_PIDFILE ]; then
pid="$(cat $MMM_AGENTD_PIDFILE)"
cnt=0
kill "$pid"
while kill -0 "$pid" 2>/dev/null; do
cnt=`expr "$cnt" + 1`
if [ "$cnt" -gt 15 ]; then
kill -9 "$pid"
break
fi
sleep 2
echo -n "."
done
echo " Ok"
exit 0
fi
echo " not running."
exit 0
;;
status)
echo -n "Checking MMM Agent process:"
if [ ! -s $MMM_AGENTD_PIDFILE ]; then
echo " not running."
exit 3
fi
pid="$(cat $MMM_AGENTD_PIDFILE)"
if ! kill -0 "$pid" 2> /dev/null; then
echo " not running."
exit 1
fi
echo " running."
exit 0
;;
restart|reload)
$0 stop
$0 start
exit $?
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
;;
esac
exit 1
mysql-mmm-2.2.1/etc/init.d/mysql-mmm-monitor 0000755 0000000 0000000 00000004455 11370745632 017467 0 ustar root root #!/bin/sh
#
# mysql-mmm-monitor This shell script takes care of starting and stopping
# the mmm monitoring daemon.
#
# chkconfig: - 64 36
# description: MMM Monitor.
# processname: mmm_mond
# config: /etc/mmm_mon.conf
# pidfile: /var/run/mmm_mond.pid
# Cluster name (it can be empty for default cases)
CLUSTER=''
#-----------------------------------------------------------------------
# Paths
if [ "$CLUSTER" != "" ]; then
MMM_MOND_BIN="/usr/sbin/mmm_mond @$CLUSTER"
MMM_MOND_PIDFILE="/var/run/mmm_mond-$CLUSTER.pid"
else
MMM_MOND_BIN="/usr/sbin/mmm_mond"
MMM_MOND_PIDFILE="/var/run/mmm_mond.pid"
fi
echo "Daemon bin: '$MMM_MOND_BIN'"
echo "Daemon pid: '$MMM_MOND_PIDFILE'"
#-----------------------------------------------------------------------
# See how we were called.
case "$1" in
start)
# Start daemon.
echo -n "Starting MMM Monitor daemon: "
if [ -s $MMM_MOND_PIDFILE ] && kill -0 `cat $MMM_MOND_PIDFILE` 2> /dev/null; then
echo " already running."
exit 0
fi
$MMM_MOND_BIN
if [ "$?" -ne 0 ]; then
echo "failed"
exit 1
fi
echo "Ok"
exit 0
;;
stop)
# Stop daemon.
echo -n "Shutting down MMM Monitor daemon: "
if [ -s $MMM_MOND_PIDFILE ]; then
pid="$(cat $MMM_MOND_PIDFILE)"
cnt=0
kill "$pid"
while kill -0 "$pid" 2>/dev/null; do
cnt=`expr "$cnt" + 1`
if [ "$cnt" -gt 15 ]; then
kill -9 "$pid"
break
fi
sleep 2
echo -n "."
done
echo " Ok"
exit 0
fi
echo " not running."
exit 0
;;
status)
echo -n "Checking MMM Monitor process:"
if [ ! -s $MMM_MOND_PIDFILE ]; then
echo " not running."
exit 3
fi
pid="$(cat $MMM_MOND_PIDFILE)"
if ! kill -0 "$pid" 2> /dev/null; then
echo " not running."
exit 1
fi
echo " running."
exit 0
;;
restart|reload)
$0 stop
$0 start
exit $?
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
;;
esac
exit 1
mysql-mmm-2.2.1/bin/ 0000755 0000000 0000000 00000000000 11370745632 012723 5 ustar root root mysql-mmm-2.2.1/bin/monitor/ 0000755 0000000 0000000 00000000000 11370745632 014412 5 ustar root root mysql-mmm-2.2.1/bin/monitor/checker 0000755 0000000 0000000 00000006445 11370745632 015755 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use English qw( OUTPUT_AUTOFLUSH );
use Log::Log4perl qw(:easy);
# TODO configurable logging via MMM::Common::Log
# Include parts of the system
use MMM::Common::Config;
use MMM::Monitor::Checker::Checks;
# Disable output buffering
$OUTPUT_AUTOFLUSH = 1;
# Check if cluster was passed
my $postfix = "";
if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) {
shift(@ARGV);
$postfix = "_$1";
}
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Fetch arguments
my $check_name = shift;
# Read config file
our $config = new MMM::Common::Config::;
$config->read("mmm_mon$postfix");
$config->check('MONITOR');
our $check;
# NOTE: the check "ping_ip" is not a host check. Its a checker for pinging single IPs to test if the monitor network connection is still working.
if ($check_name eq 'ping_ip') {
$check = {
restart_after => 0,
timeout => 1
}
}
else {
LOGDIE "checker: Unknown check $check_name" unless (defined($config->{check}->{$check_name}));
$check = \%{$config->{check}->{$check_name}};
}
my $check_function;
if ($check_name eq 'ping_ip' ) { $check_function = \&MMM::Monitor::Checker::Checks::ping_ip; }
elsif ($check_name eq 'ping' ) { $check_function = \&MMM::Monitor::Checker::Checks::ping; }
elsif ($check_name eq 'mysql' ) { $check_function = \&MMM::Monitor::Checker::Checks::mysql; }
elsif ($check_name eq 'rep_backlog') { $check_function = \&MMM::Monitor::Checker::Checks::rep_backlog; }
elsif ($check_name eq 'rep_threads') { $check_function = \&MMM::Monitor::Checker::Checks::rep_threads; }
else { LOGDIE "checker: Unknown check $check_name"; }
my $max_checks = $check->{restart_after};
my $timeout = $check->{timeout};
# Workaround to prevent checker from hanging in case of unnoticed broken pipe errors
my $max_empty_commands = 100;
my $empty_commands = 0;
INFO "$check_name: Start";
# Process loop
while (!eof(STDIN)) {
# Check if it is time to die
if ($max_checks && $max_checks < 1) {
INFO "$check_name: Max checks performed, restarting...";
last;
}
# Read command
chomp(my $cmd = );
my @command = split(/\s+/, $cmd);
my $params = scalar(@command) - 1;
# Workaround to prevent checker from hanging in case of unnoticed broken pipe errors
if ($params < 0) {
if (++$empty_commands > $max_empty_commands) {
WARN "$check_name: Too many empty commands ($empty_commands) in a row - looks like pipe is broken! Exiting!";
last;
}
next;
}
$empty_commands = 0;
last if ($command[0] eq 'quit' && $params == 0);
if ($command[0] eq 'ping' && $params == 0) {
print "OK: Pong!\n";
next;
}
if ($command[0] eq 'check' && $params == 1) {
print $check_function->($timeout, $command[1]), "\n";
next;
}
print "ERROR: Invalid command '$cmd'\n";
}
INFO "$check_name: Exit";
print "OK: Finished\n";
exit(0);
__END__
=head1 NAME
checker
=head1 DESCRIPTION
B is a helper binary for B. It is called from, and communicates with B. B processes commands from STDIN and writes the results of this commands to STDOUT:
=open 4
=item ping
Check if B is still alive.
=back
=head1 USAGE
checker
=head1 EXAMPLES
checker ping
checker mysql
checker rep_backlog
checker rep_threads
mysql-mmm-2.2.1/bin/agent/ 0000755 0000000 0000000 00000000000 11370745632 014021 5 ustar root root mysql-mmm-2.2.1/bin/agent/mysql_deny_write 0000755 0000000 0000000 00000001257 11370745632 017352 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Read config file
my $config_file = shift;
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
MMM::Agent::Helpers::Actions::mysql_deny_write();
__END__
=head1 NAME
mysql_deny_write
=head1 DESCRIPTION
mysql_deny_write is a helper binary for B. It denies writes on the local MySQL server by setting global read_only to 1.
=head1 USAGE
mysql_deny_write
mysql-mmm-2.2.1/bin/agent/clear_ip 0000755 0000000 0000000 00000001162 11370745632 015525 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 3) {
print "Usage: $0 \n\n";
exit(1);
}
# Fetch arguments
my $config_file = shift;
my $if = shift;
my $ip = shift;
# Finally do the work
MMM::Agent::Helpers::Actions::clear_ip($if, $ip);
__END__
=head1 NAME
clear_ip
=head1 DESCRIPTION
clear_ip is a helper binary for B. It removes the given ip from the given interface.
=head1 USAGE
clear_ip
=head1 EXAMPLE
clear_ip eth0 192.168.0.200
mysql-mmm-2.2.1/bin/agent/configure_ip 0000755 0000000 0000000 00000001300 11370745632 016412 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 3) {
print "Usage: $0 \n\n";
exit(1);
}
# Fetch arguments
my $config_file = shift;
my $if = shift;
my $ip = shift;
# Finally do the work
MMM::Agent::Helpers::Actions::configure_ip($if, $ip);
__END__
=head1 NAME
check_ip
=head1 DESCRIPTION
configure_ip is a helper binary for B. It checks if the given ip is configured. If not, it configures it and sends arp requests to notify other hosts.
=head1 USAGE
configure_ip
=head1 EXAMPLE
configure_ip eth0 192.168.0.200
mysql-mmm-2.2.1/bin/agent/mysql_may_write 0000755 0000000 0000000 00000001231 11370745632 017171 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Read config file
my $config_file = shift;
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
MMM::Agent::Helpers::Actions::mysql_may_write();
__END__
=head1 NAME
mysql_may_write
=head1 DESCRIPTION
mysql_may_write is a helper binary for B. It checks if writes on the local MySQL server are allowed.
=head1 USAGE
mysql_may_write
mysql-mmm-2.2.1/bin/agent/turn_off_slave 0000755 0000000 0000000 00000001273 11370745632 016766 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Read config file
my $config_file = shift;
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
my $output = MMM::Agent::Helpers::Actions::toggle_slave(0);
print $output, "\n";
exit(0);
__END__
=head1 NAME
turn_off_slave
=head1 DESCRIPTION
turn_off_slave is a helper binary for B. It stops the slave threads on the local MySQL server.
=head1 USAGE
turn_off_slave
mysql-mmm-2.2.1/bin/agent/sync_with_master 0000755 0000000 0000000 00000001547 11370745632 017340 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Read config file
my $config_file = shift;
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
my $output = MMM::Agent::Helpers::Actions::sync_with_master();
print $output, "\n";
exit(0);
__END__
=head1 NAME
sync_with_master
=head1 DESCRIPTION
sync_with_master is a helper binary for B. It tries to sync up a (soon active) master with his peer (old active master) when the I is moved. If the peer is reachable it syncs with the master log. If not reachable, syncs with the relay log.
=head1 USAGE
sync_with_master
mysql-mmm-2.2.1/bin/agent/check_ip 0000755 0000000 0000000 00000001151 11370745632 015512 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 3) {
print "Usage: $0 \n\n";
exit(1);
}
# Fetch arguments
my $config_file = shift;
my $if = shift;
my $ip = shift;
# Finally do the work
MMM::Agent::Helpers::Actions::check_ip($if, $ip);
__END__
=head1 NAME
check_ip
=head1 DESCRIPTION
check_ip is a helper binary for B. It checks if the given ip is configured.
=head1 USAGE
check_ip
=head1 EXAMPLE
check_ip eth0 192.168.0.200
mysql-mmm-2.2.1/bin/agent/turn_on_slave 0000755 0000000 0000000 00000001271 11370745632 016626 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Read config file
my $config_file = shift;
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
my $output = MMM::Agent::Helpers::Actions::toggle_slave(1);
print $output, "\n";
exit(0);
__END__
=head1 NAME
turn_on_slave
=head1 DESCRIPTION
turn_on_slave is a helper binary for B. It starts the slave threads on the local MySQL server.
=head1 USAGE
turn_on_slave
mysql-mmm-2.2.1/bin/agent/set_active_master 0000755 0000000 0000000 00000001667 11370745632 017462 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 2) {
print "Usage: $0 \n\n";
exit(1);
}
my $config_file = shift;
my $new_master = shift;
# Read config file
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
my $output = MMM::Agent::Helpers::Actions::set_active_master($new_master);
print $output, "\n";
exit(0);
__END__
=head1 NAME
set_active_master
=head1 DESCRIPTION
set_active_master is a helper binary for B. It tries to catch up with the old master as far as possible and changes the master to the new host.
(Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.)
=head1 USAGE
set_active_master
=head1 EXAMPLE
set_active_master db2
mysql-mmm-2.2.1/bin/agent/mysql_allow_write 0000755 0000000 0000000 00000001264 11370745632 017527 0 ustar root root #!/usr/bin/env perl
# Use mandatory external modules
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Agent::Helpers::Actions;
# Check arguments
if (scalar(@ARGV) != 1) {
print "Usage: $0 \n\n";
exit(1);
}
# Read config file
my $config_file = shift;
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('AGENT');
# Finally do the work
MMM::Agent::Helpers::Actions::mysql_allow_write();
__END__
=head1 NAME
mysql_allow_write
=head1 DESCRIPTION
mysql_allow_write is a helper binary for B. It allowes writes on the local MySQL server by setting global read_only to 0.
=head1 USAGE
mysql_allow_write
mysql-mmm-2.2.1/bin/tools/ 0000755 0000000 0000000 00000000000 11370745632 014063 5 ustar root root mysql-mmm-2.2.1/bin/tools/create_snapshot 0000755 0000000 0000000 00000004066 11370745632 017201 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use DBI;
use MMM::Common::Config;
use MMM::Tools::Snapshot::LVM;
use MMM::Tools::Snapshot::MySQL;
our $config = new MMM::Common::Config::;
$config->read("mmm_tools");
$config->check('TOOLS');
print create_snapshot(), "\n";
exit(0);
sub create_snapshot {
my $this = $config->{this};
unless (defined($config->{host}->{$this})) {
return "ERROR: Invalid 'this' value: '$this'!";
}
my $host = $config->{host}->{$this};
my $dump_dir = $host->{lvm_mount_dir};
system ('mkdir', '-p', $dump_dir);
unless (-d $dump_dir && -w _ && -r _ && -x _) {
return "ERROR: Directory '$dump_dir' has invalid permissions (it must be readable/writable/executable)";
}
# Check mount dir
if (scalar(glob("$dump_dir/*"))) {
return "ERROR: LVM mount dir is not empty!";
}
my $dbh = MMM::Tools::Snapshot::MySQL::connect($this);
return "ERROR: Can't connect to database! Error = " . DBI::errstr unless ($dbh);
my $res;
# Lock tables
$res = MMM::Tools::Snapshot::MySQL::lock_tables($dbh);
return "ERROR: Can't lock tables! Error = " . $dbh->errstr unless ($res);
# Get position info
my $pos_info = {};
$pos_info->{host} = $config->{this};
$res = MMM::Tools::Snapshot::MySQL::get_pos_info($dbh, $pos_info);
return "ERROR: Can't get position info: $res" unless ($res =~ /^OK/);
# Create and mount snapshot
$res = MMM::Tools::Snapshot::LVM::create();
return "ERROR: Can't create or mount snapshot: $res" unless ($res =~ /^OK/);
# Unlock tables
MMM::Tools::Snapshot::MySQL::unlock_tables($dbh);
# Change dir to snapshot and create _mmm directory
chdir($dump_dir);
system('mkdir -p _mmm');
MMM::Tools::Snapshot::MySQL::save_pos_info($pos_info, '_mmm/status.txt');
$res = system('cp', $host->{mysql_cnf}, '_mmm/');
return "ERROR: Can't copy mysql config file to backup!" if ($res);
return 'OK: Snapshot created!';
}
__END__
=head1 NAME
remove_snapshot
=head1 DESCRIPTION
remove_snapshot is a helper binary for the mmm tools. It removes a snapshot created by B.
=head1 USAGE
remove_snapshot
mysql-mmm-2.2.1/bin/tools/remove_snapshot 0000755 0000000 0000000 00000000730 11370745632 017225 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use MMM::Common::Config;
use MMM::Tools::Snapshot::LVM;
our $config = new MMM::Common::Config::;
$config->read("mmm_tools");
$config->check('TOOLS');
print MMM::Tools::Snapshot::LVM::remove(), "\n";
exit(0);
__END__
=head1 NAME
remove_snapshot
=head1 DESCRIPTION
remove_snapshot is a helper binary for the mmm tools. It removes a snapshot created by B.
=head1 USAGE
remove_snapshot
mysql-mmm-2.2.1/UPGRADE 0000644 0000000 0000000 00000004441 11370745632 013170 0 ustar root root ===== Upgrading from 2.0.x to 2.1.0 =====
MMM 2.1.0 is backwards compatible with MMM 2.0.x, so a monitoring host running
version 2.1.0 can talk to agents running 2.0.x and the other way around
==== On the monitoring host ====
1. Stop mmmd_mon (if you have multiple clusters running, be sure to stop all
instances of mmmd_mon):
/etc/init.d/mysql-mmm-monitor stop
2. When using the .debs edit /etc/default/mysql-mmm-monitor and set ENABLED
to 0.
3. Install the new version of MMM.
4. mmmd_mon was renamed to mmm_mond. So you can delete the old binary:
rm -f /usr/sbin/mmmd_mon
5. If you have multiple clusters running, recreate your init-scripts for these
by cloning /etc/init.d/mysql-mmm-monitor and adjusting the variable CLUSTER
6. (Optional) Edit mmm_mon.conf and change status_path from
/var/lib/misc/mmmd_mon.status to /var/lib/misc/mmm_mond.status - afterwards
copy the existing status file to its new location:
cp /var/lib/misc/mmmd_mon.status /var/lib/misc/mmm_mond.status
7. Edit mmm_mon.conf and change pid_path from /var/run/mmmd_mon.pid to
/var/run/mmm_mond.pid
8. The helper binaries have been moved so edit mmm_*.conf and change bin_path
from /usr/bin/mysql-mmm/ to /usr/lib/mysql-mmm/ - then delete the old
binaries:
rm -rf /usr/bin/mysql-mmm
9. When using the .debs edit /etc/default/mysql-mmm-monitor and set ENABLED
to 1.
10. Start mmm_mond (if you use more than one cluster, don't forget to start them
too):
/etc/init.d/mysql-mmm-monitor start
==== On agent hosts ====
1. Stop mmmd_agent:
/etc/init.d/mysql-mmm-agent stop
2. When using the .debs edit /etc/default/mysql-mmm-agent and set ENABLED to 0
3. Install the new version of MMM.
4. mmmd_agent was renamed to mmm_agentd. So you can delete the old binary:
rm -f /usr/sbin/mmmd_agent
5. The helper binaries have been moved so edit mmm_*.conf and change bin_path
from /usr/bin/mysql-mmm/ to /usr/lib/mysql-mmm/ - then delete the old
binaries:
rm -rf /usr/bin/mysql-mmm
6. Edit mmm_*.conf and change pid_path in the sections from
/var/run/mmmd_agent.pid to /var/run/mmm_agentd.pid
7. When using the .debs edit /etc/default/mysql-mmm-agent and set ENABLED to 1
8. Start mmm_agentd:
/etc/init.d/mysql-mmm-agent start
mysql-mmm-2.2.1/Makefile 0000644 0000000 0000000 00000003011 11370745632 013606 0 ustar root root
ifndef INSTALLDIR
INSTALLDIR = installvendorlib
endif
MODULEDIR = $(DESTDIR)$(shell eval "`perl -V:${INSTALLDIR}`"; echo "$$${INSTALLDIR}")/MMM
BINDIR = $(DESTDIR)/usr/lib/mysql-mmm
SBINDIR = $(DESTDIR)/usr/sbin
LOGDIR = $(DESTDIR)/var/log/mysql-mmm
ETCDIR = $(DESTDIR)/etc
CONFDIR = $(ETCDIR)/mysql-mmm
install_common:
mkdir -p $(DESTDIR) $(MODULEDIR) $(BINDIR) $(SBINDIR) $(LOGDIR) $(ETCDIR) $(CONFDIR) $(ETCDIR)/init.d/
cp -r lib/Common/ $(MODULEDIR)
[ -f $(CONFDIR)/mmm_common.conf ] || cp etc/mysql-mmm/mmm_common.conf $(ETCDIR)/mysql-mmm/
install_agent: install_common
mkdir -p $(BINDIR)/agent/
cp -r lib/Agent/ $(MODULEDIR)
cp -r bin/agent/* $(BINDIR)/agent/
cp -r etc/init.d/mysql-mmm-agent $(ETCDIR)/init.d/
cp sbin/mmm_agentd $(SBINDIR)
[ -f $(CONFDIR)/mmm_agent.conf ] || cp etc/mysql-mmm/mmm_agent.conf $(ETCDIR)/mysql-mmm/
install_monitor: install_common
mkdir -p $(BINDIR)/monitor/
cp -r lib/Monitor/ $(MODULEDIR)
cp -r bin/monitor/* $(BINDIR)/monitor/
cp -r etc/init.d/mysql-mmm-monitor $(ETCDIR)/init.d/
cp sbin/mmm_control sbin/mmm_mond $(SBINDIR)
[ -f $(CONFDIR)/mmm_mon.conf ] || cp etc/mysql-mmm/mmm_mon.conf $(ETCDIR)/mysql-mmm/
install_tools: install_common
mkdir -p $(BINDIR)/tools/
cp -r lib/Tools/ $(MODULEDIR)
cp -r bin/tools/* $(BINDIR)/tools/
cp sbin/mmm_backup sbin/mmm_clone sbin/mmm_restore $(SBINDIR)
[ -f $(CONFDIR)/mmm_tools.conf ] || cp etc/mysql-mmm/mmm_tools.conf $(ETCDIR)/mysql-mmm/
install: install_agent install_monitor install_tools
mysql-mmm-2.2.1/COPYING 0000644 0000000 0000000 00000043103 11370745632 013207 0 ustar root root GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users. This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it. (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.) You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.
To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have. You must make sure that they, too, receive or can get the
source code. And you must show them these terms so they know their
rights.
We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.
Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software. If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.
Finally, any free program is threatened constantly by software
patents. We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary. To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.
The precise terms and conditions for copying, distribution and
modification follow.
GNU GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a) You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b) You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c) If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a) Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b) Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c) Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
Copyright (C)
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
Also add information on how to contact you by electronic and paper mail.
If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:
Gnomovision version 69, Copyright (C) year name of author
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
`Gnomovision' (which makes passes at compilers) written by James Hacker.
, 1 April 1989
Ty Coon, President of Vice
This General Public License does not permit incorporating your program into
proprietary programs. If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License.
mysql-mmm-2.2.1/lib/ 0000755 0000000 0000000 00000000000 11370745632 012721 5 ustar root root mysql-mmm-2.2.1/lib/Common/ 0000755 0000000 0000000 00000000000 11370745632 014151 5 ustar root root mysql-mmm-2.2.1/lib/Common/Log.pm 0000644 0000000 0000000 00000002644 11370745632 015236 0 ustar root root package MMM::Common::Log;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use English qw( PROGRAM_NAME );
our $VERSION = '0.01';
sub init($$) {
my $file = shift;
my $progam = shift;
my @paths = qw(/etc /etc/mmm /etc/mysql-mmm);
# Determine filename
my $fullname;
foreach my $path (@paths) {
if (-r "$path/$file") {
$fullname = "$path/$file";
last;
}
}
# Read configuration from file
if ($fullname) {
Log::Log4perl->init($fullname);
return;
}
# Use default configuration
my $conf = "
log4perl.logger = INFO, LogFile
log4perl.appender.LogFile = Log::Log4perl::Appender::File
log4perl.appender.LogFile.Threshold = INFO
log4perl.appender.LogFile.filename = /var/log/mysql-mmm/$progam.log
log4perl.appender.LogFile.recreate = 1
log4perl.appender.LogFile.layout = PatternLayout
log4perl.appender.LogFile.layout.ConversionPattern = %d %5p %m%n
";
Log::Log4perl->init(\$conf);
}
sub debug() {
my $stdout_appender = Log::Log4perl::Appender->new(
'Log::Log4perl::Appender::Screen',
name => 'ScreenLog',
stderr => 0
);
my $layout = Log::Log4perl::Layout::PatternLayout->new('%d %5p %m%n');
$stdout_appender->layout($layout);
Log::Log4perl::Logger->get_root_logger()->add_appender($stdout_appender);
Log::Log4perl::Logger->get_root_logger()->level($DEBUG);
}
1;
mysql-mmm-2.2.1/lib/Common/Config.pm 0000644 0000000 0000000 00000042263 11370745632 015723 0 ustar root root package MMM::Common::Config;
use strict;
use warnings FATAL => 'all';
use English qw( NR );
use Log::Log4perl qw(:easy);
use List::Util qw(first);
use File::stat qw();
our $VERSION = '0.01';
# TODO remember which config file was read
our $RULESET = {
'this' => { 'required' => ['AGENT', 'TOOLS'], 'refvalues' => 'host' },
'debug' => { 'default' => 0, 'boolean' => 1 },
'active_master_role' => { 'required' => ['AGENT', 'MONITOR'], 'refvalues' => 'role' },
'max_kill_retries' => { 'default' => 10, 'required' => ['AGENT'] },
'default_copy_method' => { 'required' => ['TOOLS'], 'refvalues' => 'copy_method' },
'clone_dirs' => { 'required' => ['TOOLS'], 'multiple' => 1 },
'role' => { 'required' => ['AGENT', 'MONITOR'], 'multiple' => 1, 'section' => {
'mode' => { 'required' => ['MONITOR'], 'values' => ['balanced', 'exclusive'] },
'hosts' => { 'required' => ['MONITOR'], 'refvalues' => 'host', 'multiple' => 1 },
'ips' => { 'required' => ['AGENT', 'MONITOR'], 'multiple' => 1 },
'prefer' => { 'refvalues' => 'hosts' }
}
},
'monitor' => { 'required' => ['MONITOR', 'CONTROL'], 'section' => {
'ip' => { 'required' => ['MONITOR', 'CONTROL'] },
'port' => { 'default' => '9988' },
'pid_path' => { 'required' => ['MONITOR'] },
'bin_path' => { 'required' => ['MONITOR'] },
'status_path' => { 'required' => ['MONITOR'] },
'ping_interval' => { 'default' => 1 },
'ping_ips' => { 'required' => ['MONITOR'], 'multiple' => 1 },
'flap_duration' => { 'default' => 60 * 60 },
'flap_count' => { 'default' => 3 },
'auto_set_online' => { 'default' => 0 },
'kill_host_bin' => { 'default' => 'kill_host' },
'careful_startup' => { 'default' => 1, 'boolean' => 1 },
'mode' => { 'default' => 'active', 'values' => ['passive', 'active', 'manual', 'wait'] },
'wait_for_other_master' => { 'default' => 120 }
}
},
'socket' => { 'create_if_empty' => ['AGENT', 'CONTROL', 'MONITOR'], 'section' => {
'type' => { 'default' => 'plain', 'required' => ['AGENT', 'CONTROL', 'MONITOR'], 'values' => [ 'plain', 'ssl' ] },
'cert_file' => { 'deprequired' => { 'type' => 'ssl' }, 'required' => [ 'AGENT', 'CONTROL', 'MONITOR'] },
'key_file' => { 'deprequired' => { 'type' => 'ssl' }, 'required' => [ 'AGENT', 'CONTROL', 'MONITOR'] },
'ca_file' => { 'deprequired' => { 'type' => 'ssl' }, 'required' => [ 'AGENT', 'MONITOR'] }
}
},
'copy_method' => { 'required' => ['TOOLS'], 'multiple' => 1, 'template' => 'default', 'section' => {
'backup_command' => { 'required' => 1 },
'restore_command' => { 'required' => 1 },
'incremental_command' => { 'deprequired' => { 'incremental' => 1 } },
'incremental' => { 'default' => 0, 'boolean' => 1 },
'single_run' => { 'default' => 0, 'boolean' => 1 },
'true_copy' => { 'default' => 0, 'boolean' => 1 },
}
},
'host' => { 'required' => 1, 'multiple' => 1, 'template' => 'default', 'section' => {
'ip' => { 'required' => ['AGENT', 'MONITOR', 'TOOLS'] },
'mode' => { 'required' => ['AGENT', 'MONITOR'], 'values' => ['master', 'slave'] },
'peer' => { 'deprequired' => { 'mode' => 'master' }, 'refvalues' => 'host' },
'pid_path' => { 'required' => ['AGENT'] },
'bin_path' => { 'required' => ['AGENT'] },
'agent_port' => { 'default' => 9989 },
'cluster_interface' => { 'required' => ['AGENT'] },
'mysql_port' => { 'default' => 3306 },
'mysql_pidfile' => { 'default' => '/var/run/mysqld/mysqld.pid' },
'mysql_rcscript' => { 'default' => '/etc/init.d/mysql' },
'mysql_cnf' => { 'default' => '/etc/my.cnf' },
'agent_user' => { 'required' => ['AGENT'] },
'agent_password' => { 'required' => ['AGENT'] },
'monitor_user' => { 'required' => ['MONITOR'] },
'monitor_password' => { 'required' => ['MONITOR'] },
'replication_user' => { 'required' => ['AGENT', 'TOOLS'] },
'replication_password' => { 'required' => ['AGENT', 'TOOLS'] },
'ssh_user' => { 'required' => ['TOOLS'] },
'ssh_port' => { 'default' => 22 },
'ssh_parameters' => { 'default' => '' },
'tools_user' => { 'required' => ['TOOLS'] },
'tools_password' => { 'required' => ['TOOLS'] },
'backup_dir' => { 'required' => ['TOOLS'] },
'restore_dir' => { 'required' => ['TOOLS'] },
'lvm_bin_lvcreate' => { 'default' => 'lvcreate' },
'lvm_bin_lvremove' => { 'default' => 'lvremove' },
'lvm_snapshot_size' => { 'required' => ['TOOLS'] },
'lvm_logical_volume' => { 'required' => ['TOOLS'] },
'lvm_volume_group' => { 'required' => ['TOOLS'] },
'lvm_mount_dir' => { 'required' => ['TOOLS'] },
'lvm_mount_opts' => { 'required' => ['TOOLS'] },
}
},
'check' => { 'create_if_empty' => ['MONITOR'], 'multiple' => 1, 'template' => 'default', 'values' => ['ping', 'mysql', 'rep_backlog', 'rep_threads'], 'section' => {
'check_period' => { 'default' => 5 },
'trap_period' => { 'default' => 10 },
'timeout' => { 'default' => 2 },
'restart_after' => { 'default' => 10000 },
'max_backlog' => { 'default' => 60 } # XXX ugly
}
}
};
#-------------------------------------------------------------------------------
sub new($) {
my $self = shift;
return bless { }, $self;
}
#-------------------------------------------------------------------------------
sub read($$) {
my $self = shift;
my $file = shift;
my $fullname = $self->_get_filename($file);
LOGDIE "Could not find a readable config file" unless $fullname;
DEBUG "Loading configuration from $fullname";
my $st = File::stat::stat($fullname);
LOGDIE sprintf("Configuration file %s is world writable!", $fullname) if ($st->mode & 0002);
LOGDIE sprintf("Configuration file %s is world readable!", $fullname) if ($st->mode & 0004);
my $fd;
open($fd, "<$fullname") || LOGDIE "Can't read config file '$fullname'";
my $ret = $self->parse($RULESET, $fullname, $fd);
close($fd);
return $ret;
}
#-------------------------------------------------------------------------------
sub parse(\%\%$*); # needed because parse is a recursive function
sub parse(\%\%$*) {
my $config = shift;
my $ruleset = shift;
my $file = shift; # name of file
my $fd = shift;
my $line;
while ($line = <$fd>) {
chomp($line);
# comments and empty lines handling
next if ($line =~ /^\s*#/ || $line =~ /^\s*$/);
# end tag
return if ($line =~ /^\s*<\/\s*(\w+)\s*>\s*$/);
if ($line =~ /^\s*include\s+(\S+)\s*$/) {
my $include_file = $1;
$config->read($include_file);
next;
}
# start tag - unique section
if ($line =~/^\s*<\s*(\w+)\s*>\s*$/) {
my $type = $1;
if (!defined($ruleset->{$type}) || !defined($ruleset->{$type}->{section})) {
LOGDIE "Invalid section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
if ($ruleset->{$type}->{multiple}) {
LOGDIE "No section name specified for named section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
$config->{$type} = {} unless $config->{$type};
parse(%{$config->{$type}}, %{$ruleset->{$type}->{section}}, $file, $fd);
next;
}
# empty tag - unique section
if ($line =~/^\s*<\s*(\w+)\s*\/>\s*$/) {
my $type = $1;
if (!defined($ruleset->{$type}) || !defined($ruleset->{$type}->{section})) {
LOGDIE "Invalid section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
if ($ruleset->{$type}->{multiple}) {
LOGDIE "No section name specified for named section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
$config->{$type} = {} unless $config->{$type};
next;
}
# start tag - named section
if ($line =~/^\s*<\s*(\w+)\s+([\w\-_]+)\s*>\s*$/) {
my $type = $1;
my $name = $2;
if (!defined($ruleset->{$type}) || !defined($ruleset->{$type}->{section})) {
LOGDIE "Invalid section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
if (!$ruleset->{$type}->{multiple}) {
LOGDIE "Section name specified for unique section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
$config->{$type} = {} unless $config->{$type};
$config->{$type}->{$name} = {} unless $config->{$type}->{$name};
parse(%{$config->{$type}->{$name}}, %{$ruleset->{$type}->{section}}, $file, $fd);
next;
}
# empty tag - named section
if ($line =~/^\s*<\s*(\w+)\s+([\w\-_]+)\s*\/>\s*$/) {
my $type = $1;
my $name = $2;
if (!defined($ruleset->{$type}) || !defined($ruleset->{$type}->{section})) {
LOGDIE "Invalid section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
if (!$ruleset->{$type}->{multiple}) {
LOGDIE "Section name specified for unique section $type in '$file' on line $INPUT_LINE_NUMBER!";
}
$config->{$type} = {} unless $config->{$type};
$config->{$type}->{$name} = {} unless $config->{$type}->{$name};
next;
}
if ($line =~/^\s*(\S+)\s+(.*)$/) {
my $var = $1;
my $val = $2;
LOGDIE "Unknown variable $var in '$file' on line $INPUT_LINE_NUMBER!" unless defined($ruleset->{$var});
LOGDIE "'$var' should be a section instead of a variable in '$file' on line $INPUT_LINE_NUMBER!" if defined($ruleset->{$var}->{section});
$val =~ s/\s+$//;
@{$config->{$var}} = split(/\s*,\s*/, $val) if ($ruleset->{$var}->{multiple});
$config->{$var} = $val unless ($ruleset->{$var}->{multiple});
if ($ruleset->{$var}->{boolean}) {
$ruleset->{$var}->{values} = [0, 1];
if ($config->{$var} =~ /^(false|off|no|0)$/i) {
$config->{$var} = 0;
}
elsif ($config->{$var} =~ /^(true|on|yes|1)$/i) {
$config->{$var} = 1;
}
}
next;
}
LOGDIE "Invalid config line in file '$file' on line $INPUT_LINE_NUMBER!";
}
}
#-------------------------------------------------------------------------------
sub _get_filename($$) {
my $self = shift;
my $file = shift;
$file .= '.conf' unless ($file =~ /\.conf$/);
my @paths = qw(/etc /etc/mmm /etc/mysql-mmm);
my $fullname;
foreach my $path (@paths) {
if (-r "$path/$file") {
$fullname = "$path/$file";
last;
}
}
FATAL "No readable config file $file in ", join(', ', @paths) unless $fullname;
return $fullname;
}
#-------------------------------------------------------------------------------
sub check($$) {
my $self = shift;
my $program = shift;
$self->_check_ruleset('', $program, $RULESET, $self);
}
#-------------------------------------------------------------------------------
sub _check_ruleset(\%$$\%\%) {
my $self = shift;
my $posstr = shift;
my $program = shift;
my $ruleset = shift;
my $config = shift;
foreach my $varname (keys(%{$ruleset})) {
$self->_check_rule($posstr . $varname, $program, $ruleset, $config, $varname);
}
}
#-------------------------------------------------------------------------------
sub _check_rule(\%$$\%\%$) {
my $self = shift;
my $posstr = shift;
my $program = shift;
my $ruleset = shift;
my $config = shift;
my $varname = shift;
my $cur_rule = \%{$ruleset->{$varname}};
# set default value if not defined
if (!defined($config->{$varname}) && defined($cur_rule->{default})) {
DEBUG "Undefined value for '$posstr', using default value '$cur_rule->{default}'";
$config->{$varname} = $cur_rule->{default};
}
# check required
if (defined($cur_rule->{required}) && defined($cur_rule->{deprequired})) {
LOGDIE "Default value specified for required config entry '$posstr'" if defined($cur_rule->{default});
LOGDIE "Invalid ruleset '$posstr' - deprequired should be a hash" if (ref($cur_rule->{deprequired}) ne "HASH");
$cur_rule->{required} = _eval_program_condition($program, $cur_rule->{required}) if (ref($cur_rule->{required}) eq "ARRAY");
my ($var, $val) = %{ $cur_rule->{deprequired} };
# TODO WARN if field $var has a default value - this may not be evaluated yet.
if (!defined($config->{$varname}) && $cur_rule->{required} == 1 && defined($config->{$var}) && $config->{$var} eq $val) {
# TODO better error message for missing sections
FATAL "Config entry '$posstr' is required because of '$var $val', but missing";
}
}
elsif (defined($cur_rule->{required})) {
$cur_rule->{required} = _eval_program_condition($program, $cur_rule->{required}) if (ref($cur_rule->{required}) eq "ARRAY");
if (!defined($config->{$varname}) && $cur_rule->{required} == 1) {
# TODO better error message for sections
LOGDIE "Required config entry '$posstr' is missing";
return;
}
}
elsif (defined($cur_rule->{deprequired})) {
LOGDIE "Invalid ruleset '$posstr' - deprequired should be a hash" if (ref($cur_rule->{deprequired}) ne "HASH");
my ($var, $val) = %{ $cur_rule->{deprequired} };
# TODO WARN if field $var has a default value - this may not be evaluated yet.
if (!defined($config->{$varname}) && defined($config->{$var}) && $config->{$var} eq $val) {
# TODO better error message for missing sections
LOGDIE "Config entry '$posstr' is required because of '$var $val', but missing";
return;
}
}
return if (!defined($config->{$varname}) && !$cur_rule->{multiple});
# handle sections
if (defined($cur_rule->{section})) {
# unique secions
unless ($cur_rule->{multiple}) {
# check variables of unique sections
$self->_check_ruleset($posstr . '->', $program, $cur_rule->{section}, $config->{$varname});
return;
}
# named sections ...
# check if section name is one of the allowed
if (defined($cur_rule->{values})) {
my @allowed = @{$cur_rule->{values}};
push @allowed, $cur_rule->{template} if defined($cur_rule->{template});
foreach my $key (keys %{ $config->{$varname} }) {
unless (defined(first { $_ eq $key; } @allowed)) {
LOGDIE "Invalid $posstr '$key' in configuration allowed values are: '", join("', '", @allowed), "'"
}
}
}
# handle "create if empty"
if (defined($cur_rule->{create_if_empty})) {
$cur_rule->{create_if_empty} = _eval_program_condition($program, $cur_rule->{create_if_empty}) if (ref($cur_rule->{create_if_empty}) eq "ARRAY");
if ($cur_rule->{create_if_empty} == 1) {
$config->{$varname} = {} unless defined($config->{$varname});
foreach my $value (@{$cur_rule->{values}}) {
next if (defined($config->{$varname}->{$value}));
$config->{$varname}->{$value} = {};
}
}
}
# handle section template
if (defined($cur_rule->{template}) && defined($config->{$varname}->{ $cur_rule->{template} })) {
my $template = $config->{$varname}->{ $cur_rule->{template} };
delete($config->{$varname}->{ $cur_rule->{template} });
foreach my $var ( keys( %{ $template } ) ) {
foreach my $key ( keys( %{ $config->{$varname} } ) ) {
if (!defined($config->{$varname}->{$key}->{$var})) {
$config->{$varname}->{$key}->{$var} = $template->{$var};
}
}
}
}
# check variables of each named section
foreach my $key ( keys( %{ $config->{$varname} } ) ) {
$self->_check_ruleset($posstr . '->' . $key . '->', $program, $cur_rule->{section}, $config->{$varname}->{$key});
}
return;
}
# skip if undefined
return if (!defined($config->{$varname}));
# check if variable has one of the allowed values
if (defined($cur_rule->{values}) || defined($cur_rule->{refvalues})) {
my @allowed;
if (defined($cur_rule->{values})) {
@allowed = @{$cur_rule->{values}};
}
elsif (defined($cur_rule->{refvalues})) {
if (defined($ruleset->{ $cur_rule->{refvalues} })) {
# reference to section on current level
my $reftype = ref($config->{ $cur_rule->{refvalues} });
if ($reftype eq 'HASH') {
@allowed = keys( %{ $config->{ $cur_rule->{refvalues} } } );
# remove template section from list of valid values
if (defined($ruleset->{ $cur_rule->{refvalues} }->{template})) {
@allowed = grep { $_ ne $ruleset->{ $cur_rule->{refvalues} }->{template}} @allowed;
}
}
elsif ($reftype eq 'ARRAY') {
@allowed = @{ $config->{ $cur_rule->{refvalues} } };
}
else {
return unless (ref($config->{ $cur_rule->{refvalues} }) eq "HASH");
# LOGDIE "Could not find any $cur_rule->{refvalues}-sections";
}
}
elsif (defined($RULESET->{ $cur_rule->{refvalues} })) {
# reference to section on top level
my $reftype = ref($self->{ $cur_rule->{refvalues} });
if ($reftype eq 'HASH') {
@allowed = keys( %{ $self->{ $cur_rule->{refvalues} } } );
# remove template section from list of valid values
if (defined($RULESET->{ $cur_rule->{refvalues} }->{template})) {
@allowed = grep { $_ ne $RULESET->{ $cur_rule->{refvalues} }->{template}} @allowed;
}
}
elsif ($reftype eq 'ARRAY') {
@allowed = @{ $self->{ $cur_rule->{refvalues} } };
}
else {
return unless (ref($self->{ $cur_rule->{refvalues} }) eq "HASH");
# LOGDIE "Could not find any $cur_rule->{refvalues}-sections" unless (ref($self->{ $cur_rule->{refvalues} }) eq "HASH");
}
}
else {
LOGDIE "Invalid reference to non-section '$cur_rule->{refvalues}' for '$posstr'";
return;
}
}
if ($cur_rule->{multiple}) {
for my $val ( @{ $config->{$varname} } ) {
unless (defined(first { $_ eq $val; } @allowed)) {
LOGDIE "Config entry '$posstr' has invalid value '$val' allowed values are: '", join("', '", @allowed), "'";
return;
}
}
return;
}
unless (defined(first { $_ eq $config->{$varname}; } @allowed)) {
LOGDIE "Config entry '$posstr' has invalid value '$config->{$varname}' allowed values are: '", join("', '", @allowed), "'";
return;
}
}
}
#-------------------------------------------------------------------------------
sub _eval_program_condition($$) {
my $program = shift;
my $value = shift;
return 1 unless ($program);
return 1 if (first { $_ eq $program; } @{ $value });
return -1;
}
1;
mysql-mmm-2.2.1/lib/Common/Uptime.pm 0000644 0000000 0000000 00000001316 11370745632 015753 0 ustar root root package MMM::Common::Uptime;
use strict;
use warnings FATAL => 'all';
use English qw( OSNAME );
use Log::Log4perl qw(:easy);
require Exporter;
our @ISA = qw( Exporter );
our @EXPORT_OK = qw( uptime );
our $VERSION = '0.01';
# FIXME Solaris
if ($OSNAME eq 'linux') {
use constant UPTIME => "/proc/uptime";
}
else {
LOGDIE "Unsupported platform - can't get uptime!";
}
sub uptime {
if ($OSNAME eq 'linux') {
DEBUG "Fetching uptime from ", UPTIME;
open(FILE, UPTIME) || LOGDIE "Unable to get uptime from ", UPTIME;
my $line = ;
my ($uptime, $idle) = split(/\s+/, $line);
close(FILE);
DEBUG "Uptime is ", $uptime;
return $uptime;
}
LOGDIE "Unsupported platform - can't get uptime!";
}
1;
mysql-mmm-2.2.1/lib/Common/PidFile.pm 0000644 0000000 0000000 00000002373 11370745632 016030 0 ustar root root package MMM::Common::PidFile;
use strict;
use warnings FATAL => 'all';
use English qw( PROCESS_ID );
use Log::Log4perl qw(:easy);
our $VERSION = '0.01';
sub new($$) {
my $self = shift;
my $path = shift;
return bless { 'path' => $path }, $self;
}
sub exists($) {
my $self = shift;
return -f $self->{path};
}
sub is_running($) {
my $self = shift;
return 0 unless $self->exists();
open(PID, $self->{path}) || LOGDIE "Can't open pid file '$self->{path}' for reading!\n";
chomp(my $pid = );
close(PID);
return kill(0, $pid);
}
sub create($) {
my $self = shift;
open(PID, ">" . $self->{path}) || LOGDIE "Can't open pid file '$self->{path}' for writing!\n";
print PID $PROCESS_ID;
close(PID);
DEBUG "Created pid file '$self->{path}' with pid $PROCESS_ID";
}
sub remove($) {
my $self = shift;
unlink $self->{path};
}
1;
__END__
=head1 NAME
MMM::Common::Pidfile - Manage process id files
=cut
=head1 SYNOPSIS
my $pidfile = new MMM::Common::PidFile:: '/path/to/your.pid';
# create pidfile with current process id
$pidfile->create();
# check if pidfile exists
$pidfile->exists();
# check if the process with the process id from the pidfile is still running
$pidfile->is_running();
# remove pidfile
$pidfile->remove();
=cut
mysql-mmm-2.2.1/lib/Common/Role.pm 0000644 0000000 0000000 00000002555 11370745632 015417 0 ustar root root package MMM::Common::Role;
use strict;
use warnings FATAL => 'all';
our $VERSION = '0.01';
use Class::Struct;
use overload
'==' => \&is_equal_full,
'eq' => \&is_equal_name,
'!=' => sub { return !MMM::Common::Role::is_equal_full($_[0], $_[1]); },
'ne' => sub { return !MMM::Common::Role::is_equal_name($_[0], $_[1]); },
'cmp' => \&cmp,
'""' => \&to_string;
struct 'MMM::Common::Role' => {
name => '$',
ip => '$',
};
#-------------------------------------------------------------------------------
# NOTE: takes a role object as param
sub is_equal_full($$) {
my $self = shift;
my $other = shift;
return 0 if ($self->name ne $other->name);
return 0 if ($self->ip ne $other->ip);
return 1;
}
#-------------------------------------------------------------------------------
# NOTE: takes a role object as param
sub is_equal_name($$) {
my $self = shift;
my $other = shift;
return ($self->name eq $other->name);
}
sub cmp($$) {
my $self = shift;
my $other = shift;
return ($self->name cmp $other->name) if ($self->name ne $other->name);
return ($self->ip cmp $other->ip);
}
sub to_string($) {
my $self = shift;
return sprintf('%s(%s)', $self->name, $self->ip);
}
sub from_string($$) {
my $class = shift;
my $string = shift;
if (my ($name, $ip) = $string =~ /(.*)\((.*)\)/) {
return $class->new(name => $name, ip => $ip);
}
return undef;
}
1;
mysql-mmm-2.2.1/lib/Common/Socket.pm 0000644 0000000 0000000 00000003647 11370745632 015751 0 ustar root root package MMM::Common::Socket;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use IO::Socket::INET;
our $VERSION = '0.01';
=head1 NAME
MMM::Common::Socket - functions for socket creation
=cut
=head1 FUNCTIONS
=over 4
=item create_listener($host, $port)
Create a listening (ssl) socket on $host:$port.
=cut
sub create_listener($$) {
my $host = shift;
my $port = shift;
my $socket_class = 'IO::Socket::INET';
my %socket_opts;
my $err = sub {''};
if (defined($main::config->{'socket'}) && $main::config->{'socket'}->{type} eq 'ssl') {
require IO::Socket::SSL;
$socket_class = 'IO::Socket::SSL';
%socket_opts = (
SSL_cert_file => $main::config->{'socket'}->{cert_file},
SSL_key_file => $main::config->{'socket'}->{key_file},
SSL_ca_file => $main::config->{'socket'}->{ca_file},
SSL_verify_mode => 0x03
);
$err = sub {"\n ", IO::Socket::SSL::errstr()};
}
my $sock = $socket_class->new(
LocalHost => $host,
LocalPort => $port,
Proto => 'tcp',
Listen => 10,
Reuse => 1,
%socket_opts,
) or LOGDIE "Listener: Can't create socket!", $err->();
$sock->timeout(3);
return($sock);
}
=item create_sender($host, $port, $timeout)
Create a (ssl) client socket on $host:$port with timeout $timeout.
=cut
sub create_sender($$$) {
my $host = shift;
my $port = shift;
my $timeout = shift;
my $socket_class = 'IO::Socket::INET';
my %socket_opts;
if (defined($main::config->{'socket'}) && $main::config->{'socket'}->{type} eq "ssl") {
require IO::Socket::SSL;
$socket_class = 'IO::Socket::SSL';
%socket_opts = (
SSL_use_cert => 1,
SSL_cert_file => $main::config->{'socket'}->{cert_file},
SSL_key_file => $main::config->{'socket'}->{key_file},
SSL_ca_file => $main::config->{'socket'}->{ca_file},
);
}
return $socket_class->new(
PeerAddr => $host,
PeerPort => $port,
Proto => 'tcp',
($timeout ? (Timeout => $timeout) : ()),
%socket_opts,
);
}
1;
mysql-mmm-2.2.1/lib/Common/Angel.pm 0000644 0000000 0000000 00000004735 11370745632 015546 0 ustar root root package MMM::Common::Angel;
use strict;
use warnings FATAL => 'all';
use English qw( CHILD_ERROR ERRNO );
use Log::Log4perl qw(:easy);
use Errno qw( EINTR );
use POSIX qw( WIFEXITED WIFSIGNALED WEXITSTATUS WTERMSIG WNOHANG );
our $start_process;
our $pid;
our $attempts;
our $starttime;
sub Init($) {
my $pidfile = shift;
$MMM::Common::Angel::start_process = 1;
$MMM::Common::Angel::attempts = 0;
$MMM::Common::Angel::starttime = time();
my $is_shutdown = 0;
$pidfile->create() if (defined($pidfile));
local $SIG{INT} = \&MMM::Common::Angel::SignalHandler;
local $SIG{TERM} = \&MMM::Common::Angel::SignalHandler;
local $SIG{QUIT} = \&MMM::Common::Angel::SignalHandler;
do {
$MMM::Common::Angel::attempts++;
if ($MMM::Common::Angel::start_process) {
$MMM::Common::Angel::start_process = 0;
# Create a new child
$MMM::Common::Angel::pid = fork();
# Die if we couldn't fork
LOGDIE "Couldn't fork child process." unless (defined($MMM::Common::Angel::pid));
# Return if we are the child
return if ($MMM::Common::Angel::pid == 0);
}
# Wait for child to exit
if (waitpid($MMM::Common::Angel::pid, 0) == -1) {
if ($ERRNO{ECHLD}) {
$is_shutdown = 1 unless ($MMM::Common::Angel::start_process);
}
}
else {
if (WIFEXITED($CHILD_ERROR)) {
if (WEXITSTATUS($?) == 0) {
INFO "Child exited normally (with exitcode 0), shutting down";
$is_shutdown = 1;
}
else {
my $now = time();
my $diff = $now - $MMM::Common::Angel::starttime;
if ($MMM::Common::Angel::attempts >= 10 && $diff < 300) {
FATAL sprintf("Child exited with exitcode %s and has failed more than 10 times consecutively in the last 5 minutes, not restarting", WEXITSTATUS($?));
$MMM::Common::Angel::start_process = 0;
$is_shutdown = 1;
}
else {
FATAL sprintf("Child exited with exitcode %s, restarting after 10 second sleep", WEXITSTATUS($?));
if ($diff >= 300 ) {
# reset attempts and starttime
$MMM::Common::Angel::attempts = 0;
$MMM::Common::Angel::starttime = time();
}
sleep(10);
$MMM::Common::Angel::start_process = 1;
}
}
}
if (WIFSIGNALED($CHILD_ERROR)) {
FATAL sprintf("Child exited with signal %s, restarting", WTERMSIG($?));
$MMM::Common::Angel::start_process = 1;
}
}
} while (!$is_shutdown);
$pidfile->remove() if (defined($pidfile));
exit(0);
}
sub SignalHandler {
my $signame = shift;
kill ($signame, $MMM::Common::Angel::pid);
}
1;
mysql-mmm-2.2.1/lib/Agent/ 0000755 0000000 0000000 00000000000 11370745632 013757 5 ustar root root mysql-mmm-2.2.1/lib/Agent/Agent.pm 0000644 0000000 0000000 00000016661 11370745632 015365 0 ustar root root package MMM::Agent::Agent;
use strict;
use warnings FATAL => 'all';
use English qw(EVAL_ERROR);
use Algorithm::Diff;
use DBI;
use Class::Struct;
use Errno qw(EINTR);
use Log::Log4perl qw(:easy);
use MMM::Common::Role;
use MMM::Common::Socket;
use MMM::Agent::Helpers;
use MMM::Agent::Role;
eval {
no warnings 'once';
require Unix::Uptime;
*uptime = *Unix::Uptime->uptime;
};
if ($EVAL_ERROR) {
require MMM::Common::Uptime;
MMM::Common::Uptime->import(qw(uptime));
}
our $VERSION = '0.01';
struct 'MMM::Agent::Agent' => {
name => '$',
ip => '$',
port => '$',
interface => '$',
mode => '$',
mysql_port => '$',
mysql_user => '$',
mysql_password => '$',
writer_role => '$',
bin_path => '$',
active_master => '$',
state => '$',
roles => '@',
config_file => '$'
};
sub main($) {
my $self = shift;
my $socket = MMM::Common::Socket::create_listener($self->ip, $self->port);
$self->roles([]);
$self->active_master('');
while (!$main::shutdown) {
DEBUG 'Listener: Waiting for connection...';
my $client = $socket->accept();
next unless ($client);
DEBUG 'Listener: Connect!';
while (my $cmd = <$client>) {
chomp($cmd);
DEBUG "Daemon: Command = '$cmd'";
my $res = $self->handle_command($cmd);
my $uptime = uptime();
print $client "$res|UP:$uptime\n";
DEBUG "Daemon: Answer = '$res'";
return 0 if ($main::shutdown);
}
close($client);
DEBUG 'Listener: Disconnect!';
$self->check_roles();
}
}
sub handle_command($$) {
my $self = shift;
my $cmd = shift;
DEBUG "Received Command $cmd";
my ($cmd_name, $version, $host, @params) = split('\|', $cmd, -1);
return "ERROR: Invalid command '$cmd'!" unless (defined($host));
return "ERROR: Invalid hostname in command ($host)! My name is '" . $self->name . "'" if ($host ne $self->name);
if ($version > main::MMM_PROTOCOL_VERSION) {
WARN "Version in command '$cmd_name' ($version) is greater than mine (", main::MMM_PROTOCOL_VERSION, ")"
}
if ($cmd_name eq 'PING') { return cmd_ping (); }
elsif ($cmd_name eq 'SET_STATUS') { return $self->cmd_set_status (@params); }
elsif ($cmd_name eq 'GET_AGENT_STATUS') { return $self->cmd_get_agent_status (); }
elsif ($cmd_name eq 'GET_SYSTEM_STATUS') { return $self->cmd_get_system_status (); }
elsif ($cmd_name eq 'CLEAR_BAD_ROLES') { return $self->cmd_clear_bad_roles (); }
return "ERROR: Invalid command '$cmd_name'!";
}
sub cmd_ping() {
return 'OK: Pinged!';
}
sub cmd_get_agent_status($) {
my $self = shift;
my $answer = join ('|', (
$self->state,
join(',', @{$self->roles}),
$self->active_master
));
return "OK: Returning status!|$answer";
}
sub cmd_get_system_status($) {
my $self = shift;
# determine master info
my $dsn = sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $self->ip, $self->mysql_port);
my $eintr = EINTR;
my $master_ip = '';
my $dbh;
CONNECT: {
DEBUG "Connecting to mysql";
$dbh = DBI->connect($dsn, $self->mysql_user, $self->mysql_password, { PrintError => 0 });
unless ($dbh) {
redo CONNECT if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/);
WARN "Couldn't connect to mysql. Can't determine current master host." . $DBI::err . " " . $DBI::errstr;
}
}
my $slave_status = $dbh->selectrow_hashref('SHOW SLAVE STATUS');
$master_ip = $slave_status->{Master_Host} if (defined($slave_status));
my @roles;
foreach my $role (keys(%{$main::config->{role}})) {
my $role_info = $main::config->{role}->{$role};
foreach my $ip (@{$role_info->{ips}}) {
my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip);
my $ret = $? >> 8;
return "ERROR: Could not check if IP is configured: $res" if ($ret == 255);
next unless ($ret == 0);
# IP is configured...
push @roles, new MMM::Common::Role::(name => $role, ip => $ip);
}
}
my $res = MMM::Agent::Helpers::may_write();
my $ret = $? >> 8;
return "ERROR: Could not check if MySQL is writable: $res" if ($ret == 255);
my $writable = ($ret == 1);
my $answer = join('|', ($writable, join(',', @roles), $master_ip));
return "OK: Returning status!|$answer";
}
sub cmd_clear_bad_roles($) {
my $self = shift;
my $count = 0;
foreach my $role (keys(%{$main::config->{role}})) {
my $role_info = $main::config->{role}->{$role};
foreach my $ip (@{$role_info->{ips}}) {
my $role_valid = 0;
foreach my $agentrole (@{$self->roles}) {
next unless ($agentrole->name eq $role);
next unless ($agentrole->ip eq $ip);
$role_valid = 1;
last;
}
next if ($role_valid);
my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip);
my $ret = $? >> 8;
return "ERROR: Could not check if IP is configured: $res" if ($ret == 255);
next if ($ret == 1);
# IP is configured...
my $roleobj = new MMM::Agent::Role::(name => $role, ip => $ip);
$roleobj->del();
$count++;
}
}
return "OK: Removed $count roles";
}
sub cmd_set_status($$) {
my $self = shift;
my ($new_state, $new_roles_str, $new_master) = @_;
# Change master if we are a slave
if ($new_master ne $self->active_master && $self->mode eq 'slave' && $new_state eq 'ONLINE' && $new_master ne '') {
INFO "Changing active master to '$new_master'";
my $res = MMM::Agent::Helpers::set_active_master($new_master);
DEBUG sprintf("Result: %s", defined($res) ? $res : 'undef');
if (defined($res) && $res =~ /^OK/) {
$self->active_master($new_master);
}
else {
FATAL sprintf("Failed to change master to '%s': %s", $new_master, defined($res) ? $res : 'undef');
}
}
# Parse roles
my @new_roles_arr = sort(split(/\,/, $new_roles_str));
my @new_roles;
foreach my $role_str (@new_roles_arr) {
my $role = MMM::Agent::Role->from_string($role_str);
if (defined($role)) {
push @new_roles, $role;
}
}
# Process roles
my @added_roles = ();
my @deleted_roles = ();
my $changes_count = 0;
# Determine changes
my $diff = new Algorithm::Diff:: ($self->roles, \@new_roles, { keyGen => \&MMM::Common::Role::to_string });
while ($diff->Next) {
next if ($diff->Same);
$changes_count++;
push (@deleted_roles, $diff->Items(1)) if ($diff->Items(1));
push (@added_roles, $diff->Items(2)) if ($diff->Items(2));
}
# Apply changes
if ($changes_count) {
INFO 'We have some new roles added or old rules deleted!';
INFO 'Deleted: ', join(', ', sort(@deleted_roles)) if (scalar(@deleted_roles));
INFO 'Added: ', join(', ', sort(@added_roles)) if (scalar(@added_roles));
foreach my $role (@deleted_roles) { $role->del(); }
foreach my $role (@added_roles) { $role->add(); }
$self->roles(\@new_roles);
}
# Process state change
if ($new_state ne $self->state) {
if ($new_state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_off_slave(); }
if ($self->state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_on_slave(); }
$self->state($new_state);
}
return 'OK: Status applied successfully!';
}
sub check_roles($) {
my $self = shift;
foreach my $role (@{$self->roles}) {
$role->check();
}
}
sub from_config($%) {
my $self = shift;
my $config = shift;
my $host = $config->{host}->{$config->{this}};
$self->name ($config->{this});
$self->ip ($host->{ip});
$self->port ($host->{agent_port});
$self->interface ($host->{cluster_interface});
$self->mode ($host->{mode});
$self->mysql_port ($host->{mysql_port});
$self->mysql_user ($host->{agent_user});
$self->mysql_password ($host->{agent_password});
$self->writer_role ($config->{active_master_role});
$self->bin_path ($host->{bin_path});
}
1;
mysql-mmm-2.2.1/lib/Agent/Helpers/ 0000755 0000000 0000000 00000000000 11370745632 015361 5 ustar root root mysql-mmm-2.2.1/lib/Agent/Helpers/Actions.pm 0000644 0000000 0000000 00000031235 11370745632 017323 0 ustar root root package MMM::Agent::Helpers::Actions;
use strict;
use warnings FATAL => 'all';
use MMM::Agent::Helpers::Network;
our $VERSION = '0.01';
=head1 NAME
MMM::Agent::Helpers::Actions - functions for the B helper programs
=cut
use DBI;
=head1 FUNCTIONS
=over 4
=item check_ip($if, $ip)
Check if the IP $ip is configured on interface $if.
=cut
sub check_ip($$) {
my $if = shift;
my $ip = shift;
if (MMM::Agent::Helpers::Network::check_ip($if, $ip)) {
_exit_ok('IP address is configured');
}
_exit_ok('IP address is not configured', 1);
}
=item configure_ip($if, $ip)
Check if the IP $ip is configured on interface $if. If not, configure it and
send arp requests to notify other hosts.
=cut
sub configure_ip($$) {
my $if = shift;
my $ip = shift;
if (MMM::Agent::Helpers::Network::check_ip($if, $ip)) {
_exit_ok('IP address is configured');
}
if (!MMM::Agent::Helpers::Network::add_ip($if, $ip)) {
_exit_error("Could not configure ip adress $ip on interface $if!");
}
MMM::Agent::Helpers::Network::send_arp($if, $ip);
_exit_ok();
}
=item clear_ip($if, $ip)
Remove the IP address $ip from interface $if.
=cut
sub clear_ip($$) {
my $if = shift;
my $ip = shift;
if (!MMM::Agent::Helpers::Network::check_ip($if, $ip)) {
_exit_ok('IP address is not configured');
}
MMM::Agent::Helpers::Network::clear_ip($if, $ip);
_exit_ok();
}
=item mysql_may_write( )
Check if writes on local MySQL server are allowed.
=cut
sub mysql_may_write() {
my ($host, $port, $user, $password) = _get_connection_info();
_exit_error('No connection info') unless defined($host);
# connect to server
my $dbh = _mysql_connect($host, $port, $user, $password);
_exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh);
# check old read_only state
(my $read_only) = $dbh->selectrow_array('select @@read_only');
_exit_error('SQL Query Error: ' . $dbh->errstr) unless (defined $read_only);
_exit_ok('Not allowed') if ($read_only);
_exit_ok('Allowed', 1);
}
=item mysql_allow_write( )
Allow writes on local MySQL server. Sets global read_only to 0.
=cut
sub mysql_allow_write() {
_mysql_set_read_only(0);
_exit_ok();
}
=item mysql_deny_write( )
Deny writes on local MySQL server. Sets global read_only to 1.
=cut
sub mysql_deny_write() {
_mysql_set_read_only(1);
kill_sql();
_exit_ok();
}
sub _mysql_set_read_only($) {
my $read_only_new = shift;
my ($host, $port, $user, $password) = _get_connection_info();
_exit_error('No connection info') unless defined($host);
# connect to server
my $dbh = _mysql_connect($host, $port, $user, $password);
_exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh);
# check old read_only state
(my $read_only_old) = $dbh->selectrow_array('select @@read_only');
_exit_error('SQL Query Error: ' . $dbh->errstr) unless (defined $read_only_old);
return 1 if ($read_only_old == $read_only_new);
my $res = $dbh->do("set global read_only=$read_only_new");
_exit_error('SQL Query Error: ' . $dbh->errstr) unless($res);
$dbh->disconnect();
$dbh = undef;
return 1;
}
=item kill_sql
kill all user threads to prevent further writes
=cut
sub kill_sql() {
my ($host, $port, $user, $password) = _get_connection_info();
_exit_error('No connection info') unless defined($host);
# Connect to server
my $dbh = _mysql_connect($host, $port, $user, $password);
_exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh);
my $my_id = $dbh->{'mysql_thread_id'};
my $max_retries = $main::config->{max_kill_retries};
my $elapsed_retries = 0;
my $retry = 1;
while ($elapsed_retries <= $max_retries && $retry) {
$retry = 0;
# Fetch process list
my $processlist = $dbh->selectall_hashref('SHOW PROCESSLIST', 'Id');
# Kill processes
foreach my $id (keys(%{$processlist})) {
# Skip ourselves
next if ($id == $my_id);
# Skip non-client threads (i.e. I/O or SQL threads used on replication slaves, ...)
next if ($processlist->{$id}->{User} eq 'system user');
# skip threads of replication clients
next if ($processlist->{$id}->{Command} eq 'Binlog Dump');
# Give threads a chance to finish if we're not on our last retry
if ($elapsed_retries < $max_retries
&& defined ($processlist->{$id}->{Info})
&& $processlist->{$id}->{Info} =~ /^\s*(\/\*.*?\*\/)?\s*(INSERT|UPDATE|DELETE|REPLACE|CREATE|DROP|ALTER|REPAIR|OPTIMIZE|ANALYZE|CHECK)/si
) {
$retry = 1;
next;
}
# Kill process
$dbh->do("KILL $id");
}
sleep(1) if ($elapsed_retries < $max_retries && $retry);
$elapsed_retries++;
}
}
=item toggle_slave($state)
Toggle slave state. Starts slave if $state != 0. Stops it otherwise.
=cut
sub toggle_slave($) {
my $state = shift;
my ($host, $port, $user, $password) = _get_connection_info();
_exit_error('No connection info') unless defined($host);
my $query = $state ? 'START SLAVE' : 'STOP SLAVE';
# connect to server
my $dbh = _mysql_connect($host, $port, $user, $password);
_exit_error("Can't connect to MySQL (host = $host:$port, user = $user)! " . $DBI::errstr) unless ($dbh);
# execute query
my $res = $dbh->do($query);
_exit_error('SQL Query Error: ' . $dbh->errstr) unless($res);
_exit_ok();
}
=item sync_with_master( )
Try to sync up a (soon active) master with his peer (old active master) when the I is moved. If the peer is reachable it syncs with the master log. If not reachable, syncs with the relay log.
=cut
sub sync_with_master() {
my $this = _get_this();
my ($this_host, $this_port, $this_user, $this_password) = _get_connection_info($this);
_exit_error('No local connection info') unless defined($this_host);
my $peer = $main::config->{host}->{$this}->{peer};
_exit_error('No peer defined') unless defined($peer);
my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($peer);
_exit_error('No peer connection info') unless defined($peer_host);
# Connect to local server
my $this_dbh = _mysql_connect($this_host, $this_port, $this_user, $this_password);
_exit_error("Can't connect to MySQL (host = $this_host:$this_port, user = $this_user)! " . $DBI::errstr) unless ($this_dbh);
# Connect to peer
my $peer_dbh = _mysql_connect($peer_host, $peer_port, $peer_user, $peer_password);
# Determine wait log and wait pos
my $wait_log;
my $wait_pos;
if ($peer_dbh) {
my $master_status = $peer_dbh->selectrow_hashref('SHOW MASTER STATUS');
if (defined($master_status)) {
$wait_log = $master_status->{File};
$wait_pos = $master_status->{Position};
}
$peer_dbh->disconnect;
}
unless (defined($wait_log)) {
my $slave_status = $this_dbh->selectrow_hashref('SHOW SLAVE STATUS');
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless defined($slave_status);
$wait_log = $slave_status->{Master_Log_File};
$wait_pos = $slave_status->{Read_Master_Log_Pos};
}
# Sync with logs
my $res = $this_dbh->do("SELECT MASTER_POS_WAIT('$wait_log', $wait_pos)");
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res);
_exit_ok();
}
=item set_active_master($new_master)
Try to catch up with the old master as far as possible and change the master to the new host.
(Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.)
=cut
sub set_active_master($) {
my $new_peer = shift;
_exit_error('Name of new master is missing') unless (defined($new_peer));
my $this = _get_this();
_exit_error('New master is equal to local host!?') if ($this eq $new_peer);
# Get local connection info
my ($this_host, $this_port, $this_user, $this_password) = _get_connection_info($this);
_exit_error("No connection info for local host '$this_host'") unless defined($this_host);
# Get connection info for new peer
my ($new_peer_host, $new_peer_port, $new_peer_user, $new_peer_password) = _get_connection_info($new_peer);
_exit_error("No connection info for new peer '$new_peer'") unless defined($new_peer_host);
# Connect to local server
my $this_dbh = _mysql_connect($this_host, $this_port, $this_user, $this_password);
_exit_error("Can't connect to MySQL (host = $this_host:$this_port, user = $this_user)! " . $DBI::errstr) unless ($this_dbh);
# Get slave info
my $slave_status = $this_dbh->selectrow_hashref('SHOW SLAVE STATUS');
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless defined($slave_status);
my $wait_log = $slave_status->{Master_Log_File};
my $wait_pos = $slave_status->{Read_Master_Log_Pos};
my $old_peer_ip = $slave_status->{Master_Host};
_exit_error('No ip for old peer') unless ($old_peer_ip);
# Get connection info for old peer
my $old_peer = _find_host_by_ip($old_peer_ip);
_exit_error('Invalid master host in show slave status') unless ($old_peer);
_exit_ok('We are already a slave of the new master') if ($old_peer eq $new_peer);
my ($old_peer_host, $old_peer_port, $old_peer_user, $old_peer_password) = _get_connection_info($old_peer);
_exit_error("No connection info for new peer '$old_peer'") unless defined($old_peer_host);
my $old_peer_dbh = _mysql_connect($old_peer_host, $old_peer_port, $old_peer_user, $old_peer_password);
if ($old_peer_dbh) {
my $old_master_status = $old_peer_dbh->selectrow_hashref('SHOW MASTER STATUS');
if (defined($old_master_status)) {
$wait_log = $old_master_status->{File};
$wait_pos = $old_master_status->{Position};
}
$old_peer_dbh->disconnect;
}
# Sync with logs
my $res = $this_dbh->do("SELECT MASTER_POS_WAIT('$wait_log', $wait_pos)");
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res);
# Stop slave
$res = $this_dbh->do('STOP SLAVE');
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res);
# Connect to new peer
my $new_peer_dbh = _mysql_connect($new_peer_host, $new_peer_port, $new_peer_user, $new_peer_password);
_exit_error("Can't connect to MySQL (host = $new_peer_host:$new_peer_port, user = $new_peer_user)! " . $DBI::errstr) unless ($new_peer_dbh);
# Get log position of new master
my $new_master_status = $new_peer_dbh->selectrow_hashref('SHOW MASTER STATUS');
_exit_error('SQL Query Error: ' . $new_peer_dbh->errstr) unless($new_master_status);
my $master_log = $new_master_status->{File};
my $master_pos = $new_master_status->{Position};
$new_peer_dbh->disconnect;
# Get replication credentials
my ($repl_user, $repl_password) = _get_replication_credentials($new_peer);
# Change master
my $sql = 'CHANGE MASTER TO'
. " MASTER_HOST='$new_peer_host',"
. " MASTER_PORT=$new_peer_port,"
. " MASTER_USER='$repl_user',"
. " MASTER_PASSWORD='$repl_password',"
. " MASTER_LOG_FILE='$master_log',"
. " MASTER_LOG_POS=$master_pos";
$res = $this_dbh->do($sql);
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res);
# Start slave
$res = $this_dbh->do('START SLAVE');
_exit_error('SQL Query Error: ' . $this_dbh->errstr) unless($res);
return 'OK';
}
=item _get_connection_info([$host])
Get connection info for host $host || local host.
=cut
sub _get_connection_info($) {
my $host = shift;
_exit_error('No config present') unless (defined($main::config));
$host = $main::config->{this} unless defined($host);
_exit_error('No config present') unless (defined($main::config->{host}->{$host}));
return (
$main::config->{host}->{$host}->{ip},
$main::config->{host}->{$host}->{mysql_port},
$main::config->{host}->{$host}->{agent_user},
$main::config->{host}->{$host}->{agent_password}
);
}
sub _get_this() {
_exit_error('No config present') unless (defined($main::config));
return $main::config->{this};
}
sub _mysql_connect($$$$) {
my ($host, $port, $user, $password) = @_;
my $dsn = "DBI:mysql:host=$host;port=$port;mysql_connect_timeout=3";
return DBI->connect($dsn, $user, $password, { PrintError => 0 });
}
sub _find_host_by_ip($) {
my $ip = shift;
return undef unless ($ip);
_exit_error('No config present') unless (defined($main::config));
my $hosts = $main::config->{host};
foreach my $host (keys(%$hosts)) {
return $host if ($hosts->{$host}->{ip} eq $ip);
}
return undef;
}
sub _get_replication_credentials($) {
my $host = shift;
return undef unless ($host);
_exit_error('No config present') unless (defined($main::config));
_exit_error('No config present') unless (defined($main::config->{host}->{$host}));
return (
$main::config->{host}->{$host}->{replication_user},
$main::config->{host}->{$host}->{replication_password},
);
}
sub _exit_error {
my $msg = shift;
print "ERROR: $msg\n" if ($msg);
print "ERROR\n" unless ($msg);
exit(255);
}
sub _exit_ok {
my $msg = shift;
my $ret = shift || 0;
print "OK: $msg\n" if ($msg);
print "OK\n" unless ($msg);
exit($ret);
}
sub _verbose_exit($$) {
my $ret = shift;
my $msg = shift;
print $msg, "\n";
exit($ret);
}
1;
mysql-mmm-2.2.1/lib/Agent/Helpers/Network.pm 0000644 0000000 0000000 00000010737 11370745632 017360 0 ustar root root package MMM::Agent::Helpers::Network;
use strict;
use warnings FATAL => 'all';
use English qw( OSNAME );
our $VERSION = '0.01';
if ($OSNAME eq 'linux' || $OSNAME eq 'freebsd') {
# these libs will always be loaded, use require and then import to avoid that
use Net::ARP;
use Time::HiRes qw( usleep );
}
=head1 NAME
MMM::Agent::Helpers::Network - network related functions for the B helper programs
=cut
=head1 FUNCTIONS
=over 4
=item check_ip($if, $ip)
Check if the IP $ip is configured on interface $if. Returns 0 if not, 1 otherwise.
=cut
sub check_ip($$) {
my $if = shift;
my $ip = shift;
my $output;
if ($OSNAME eq 'linux') {
$output = `/sbin/ip addr show dev $if`;
_exit_error("Could not check if ip $ip is configured on $if: $output") if ($? >> 8 == 255);
}
elsif ($OSNAME eq 'solaris') {
# FIXME $if is not used here
$output = `/usr/sbin/ifconfig -a | grep inet`;
_exit_error("Could not check if ip $ip is configured on $if: $output") if ($? >> 8 == 255);
}
elsif ($OSNAME eq 'freebsd') {
$output = `/sbin/ifconfig $if | grep inet`;
_exit_error("Could not check if ip $ip is configured on $if: $output") if ($? >> 8 == 255);
}
else {
_exit_error("ERROR: Unsupported platform!");
}
return ($output =~ /\D+$ip\D+/) ? 1 : 0;
}
=item add_ip($if, $ip)
Add IP $ip to the interface $if.
=cut
sub add_ip($$) {
my $if = shift;
my $ip = shift;
my $output;
if ($OSNAME eq 'linux') {
$output = `/sbin/ip addr add $ip/32 dev $if`;
_exit_error("Could not configure ip $ip on interface $if: $output") if ($? >> 8 == 255);
}
elsif ($OSNAME eq 'solaris') {
$output = `/usr/sbin/ifconfig $if addif $ip`;
_exit_error("Could not configure ip $ip on interface $if: $output") if ($? >> 8 == 255);
my $logical_if = _solaris_find_logical_if($ip);
unless ($logical_if) {
_exit_error("ERROR: Can't find logical interface with IP = $ip");
}
$output = `/usr/sbin/ifconfig $logical_if up`;
_exit_error("Could not activate logical interface $logical_if with ip $ip on interface: $output") if ($? >> 8 == 255);
}
elsif ($OSNAME eq 'freebsd') {
$output = `/sbin/ifconfig $if inet $ip netmask 255.255.255.255 alias`;
_exit_error("Could not configure ip $ip on interface $if: $output") if ($? >> 8 == 255);
}
else {
_exit_error("ERROR: Unsupported platform!");
}
return check_ip($if, $ip);
}
=item clear_ip($if, $ip)
Remove the IP $ip from the interface $if.
=cut
sub clear_ip($$) {
my $if = shift;
my $ip = shift;
my $output;
if ($OSNAME eq 'linux') {
$output = `/sbin/ip addr del $ip/32 dev $if`;
_exit_error("Could not remove ip $ip from interface $if: $output") if ($? >> 8 == 255);
}
elsif ($OSNAME eq 'solaris') {
$output = `/usr/sbin/ifconfig $if removeif $ip`;
_exit_error("Could not remove ip $ip from interface $if: $output") if ($? >> 8 == 255);
}
elsif ($OSNAME eq 'freebsd') {
$output = `/sbin/ifconfig $if inet $ip -alias`;
_exit_error("Could not remove ip $ip from interface $if: $output") if ($? >> 8 == 255);
}
else {
exit(1);
}
}
=item send_arp($if, $ip)
Send arp requests for the IP $ip to the broadcast address on network interface $if.
=cut
sub send_arp($$) {
my $if = shift;
my $ip = shift;
if ($OSNAME eq 'linux' || $OSNAME eq 'freebsd') {
my $mac = '';
if ($Net::ARP::VERSION < 1.0) {
Net::ARP::get_mac($if, $mac);
}
else {
$mac = Net::ARP::get_mac($if);
}
return "ERROR: Couldn't get mac adress of interface $if" unless ($mac);
for (my $i = 0; $i < 5; $i++) {
Net::ARP::send_packet($if, $ip, $ip, $mac, 'ff:ff:ff:ff:ff:ff', 'request');
usleep(50);
Net::ARP::send_packet($if, $ip, $ip, $mac, 'ff:ff:ff:ff:ff:ff', 'reply');
usleep(50) if ($i < 4);
}
}
elsif ($OSNAME eq 'solaris') {
# Get params for send_arp
my $ipaddr = `/usr/sbin/ifconfig $if`;
# Get broadcast address and netmask
$ipaddr =~ /netmask\s*([0-9a-f]+)\s*broadcast\s*([\d\.]+)/i;
my $if_bcast = $1;
my $if_mask = $2;
`/bin/send_arp -i 100 -r 5 -p /tmp/send_arp $if $ip auto $if_bcast $if_mask`;
}
else {
_exit_error("ERROR: Unsupported platform!");
}
}
sub _exit_error {
my $msg = shift;
print "ERROR: $msg\n" if ($msg);
print "ERROR\n" unless ($msg);
exit(255);
}
#-------------------------------------------------------------------------------
sub _solaris_find_logical_if($) {
my $ip = shift;
my $ifconfig = `/usr/sbin/ifconfig -a`;
$ifconfig =~ s/\n/ /g;
while ($ifconfig =~ s/([a-z0-9\:]+)(\:\s+.*?)inet\s*([0-9\.]+)//) {
return $1 if ($3 eq $ip);
}
return undef;
}
1;
mysql-mmm-2.2.1/lib/Agent/Agent.pm.orig 0000644 0000000 0000000 00000016632 11370745632 016322 0 ustar root root package MMM::Agent::Agent;
use strict;
use warnings FATAL => 'all';
use English qw(EVAL_ERROR);
use Algorithm::Diff;
use DBI;
use Class::Struct;
use Errno qw(EINTR);
use Log::Log4perl qw(:easy);
use MMM::Common::Role;
use MMM::Common::Socket;
use MMM::Agent::Helpers;
use MMM::Agent::Role;
eval {
no warnings 'once';
require Unix::Uptime;
*uptime = *Unix::Uptime->uptime;
};
if ($EVAL_ERROR) {
require MMM::Common::Uptime;
MMM::Common::Uptime->import(qw(uptime));
}
our $VERSION = '0.01';
struct 'MMM::Agent::Agent' => {
name => '$',
ip => '$',
port => '$',
interface => '$',
mode => '$',
mysql_port => '$',
mysql_user => '$',
mysql_password => '$',
writer_role => '$',
bin_path => '$',
active_master => '$',
state => '$',
roles => '@'
};
sub main($) {
my $self = shift;
my $socket = MMM::Common::Socket::create_listener($self->ip, $self->port);
$self->roles([]);
$self->active_master('');
while (!$main::shutdown) {
DEBUG 'Listener: Waiting for connection...';
my $client = $socket->accept();
next unless ($client);
DEBUG 'Listener: Connect!';
while (my $cmd = <$client>) {
chomp($cmd);
DEBUG "Daemon: Command = '$cmd'";
my $res = $self->handle_command($cmd);
my $uptime = uptime();
print $client "$res|UP:$uptime\n";
DEBUG "Daemon: Answer = '$res'";
return 0 if ($main::shutdown);
}
close($client);
DEBUG 'Listener: Disconnect!';
$self->check_roles();
}
}
sub handle_command($$) {
my $self = shift;
my $cmd = shift;
DEBUG "Received Command $cmd";
my ($cmd_name, $version, $host, @params) = split('\|', $cmd, -1);
return "ERROR: Invalid command '$cmd'!" unless (defined($host));
return "ERROR: Invalid hostname in command ($host)! My name is '" . $self->name . "'" if ($host ne $self->name);
if ($version > main::MMM_PROTOCOL_VERSION) {
WARN "Version in command '$cmd_name' ($version) is greater than mine (", main::MMM_PROTOCOL_VERSION, ")"
}
if ($cmd_name eq 'PING') { return cmd_ping (); }
elsif ($cmd_name eq 'SET_STATUS') { return $self->cmd_set_status (@params); }
elsif ($cmd_name eq 'GET_AGENT_STATUS') { return $self->cmd_get_agent_status (); }
elsif ($cmd_name eq 'GET_SYSTEM_STATUS') { return $self->cmd_get_system_status (); }
elsif ($cmd_name eq 'CLEAR_BAD_ROLES') { return $self->cmd_clear_bad_roles (); }
return "ERROR: Invalid command '$cmd_name'!";
}
sub cmd_ping() {
return 'OK: Pinged!';
}
sub cmd_get_agent_status($) {
my $self = shift;
my $answer = join ('|', (
$self->state,
join(',', @{$self->roles}),
$self->active_master
));
return "OK: Returning status!|$answer";
}
sub cmd_get_system_status($) {
my $self = shift;
# determine master info
my $dsn = sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $self->ip, $self->mysql_port);
my $eintr = EINTR;
my $master_ip = '';
my $dbh;
CONNECT: {
DEBUG "Connecting to mysql";
$dbh = DBI->connect($dsn, $self->mysql_user, $self->mysql_password, { PrintError => 0 });
unless ($dbh) {
redo CONNECT if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/);
WARN "Couldn't connect to mysql. Can't determine current master host." . $DBI::err . " " . $DBI::errstr;
}
}
my $slave_status = $dbh->selectrow_hashref('SHOW SLAVE STATUS');
$master_ip = $slave_status->{Master_Host} if (defined($slave_status));
my @roles;
foreach my $role (keys(%{$main::config->{role}})) {
my $role_info = $main::config->{role}->{$role};
foreach my $ip (@{$role_info->{ips}}) {
my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip);
my $ret = $? >> 8;
return "ERROR: Could not check if IP is configured: $res" if ($ret == 255);
next unless ($ret == 0);
# IP is configured...
push @roles, new MMM::Common::Role::(name => $role, ip => $ip);
}
}
my $res = MMM::Agent::Helpers::may_write();
my $ret = $? >> 8;
return "ERROR: Could not check if MySQL is writable: $res" if ($ret == 255);
my $writable = ($ret == 1);
my $answer = join('|', ($writable, join(',', @roles), $master_ip));
return "OK: Returning status!|$answer";
}
sub cmd_clear_bad_roles($) {
my $self = shift;
my $count = 0;
foreach my $role (keys(%{$main::config->{role}})) {
my $role_info = $main::config->{role}->{$role};
foreach my $ip (@{$role_info->{ips}}) {
my $role_valid = 0;
foreach my $agentrole (@{$self->roles}) {
next unless ($agentrole->name eq $role);
next unless ($agentrole->ip eq $ip);
$role_valid = 1;
last;
}
next if ($role_valid);
my $res = MMM::Agent::Helpers::check_ip($self->interface, $ip);
my $ret = $? >> 8;
return "ERROR: Could not check if IP is configured: $res" if ($ret == 255);
next if ($ret == 1);
# IP is configured...
my $roleobj = new MMM::Agent::Role::(name => $role, ip => $ip);
$roleobj->del();
$count++;
}
}
return "OK: Removed $count roles";
}
sub cmd_set_status($$) {
my $self = shift;
my ($new_state, $new_roles_str, $new_master) = @_;
# Change master if we are a slave
if ($new_master ne $self->active_master && $self->mode eq 'slave' && $new_state eq 'ONLINE' && $new_master ne '') {
INFO "Changing active master to '$new_master'";
my $res = MMM::Agent::Helpers::set_active_master($new_master);
DEBUG sprintf("Result: %s", defined($res) ? $res : 'undef');
if (defined($res) && $res =~ /^OK/) {
$self->active_master($new_master);
}
else {
FATAL sprintf("Failed to change master to '%s': %s", $new_master, defined($res) ? $res : 'undef');
}
}
# Parse roles
my @new_roles_arr = sort(split(/\,/, $new_roles_str));
my @new_roles;
foreach my $role_str (@new_roles_arr) {
my $role = MMM::Agent::Role->from_string($role_str);
if (defined($role)) {
push @new_roles, $role;
}
}
# Process roles
my @added_roles = ();
my @deleted_roles = ();
my $changes_count = 0;
# Determine changes
my $diff = new Algorithm::Diff:: ($self->roles, \@new_roles, { keyGen => \&MMM::Common::Role::to_string });
while ($diff->Next) {
next if ($diff->Same);
$changes_count++;
push (@deleted_roles, $diff->Items(1)) if ($diff->Items(1));
push (@added_roles, $diff->Items(2)) if ($diff->Items(2));
}
# Apply changes
if ($changes_count) {
INFO 'We have some new roles added or old rules deleted!';
INFO 'Deleted: ', join(', ', sort(@deleted_roles)) if (scalar(@deleted_roles));
INFO 'Added: ', join(', ', sort(@added_roles)) if (scalar(@added_roles));
foreach my $role (@deleted_roles) { $role->del(); }
foreach my $role (@added_roles) { $role->add(); }
$self->roles(\@new_roles);
}
# Process state change
if ($new_state ne $self->state) {
if ($new_state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_off_slave(); }
if ($self->state eq 'ADMIN_OFFLINE') { MMM::Agent::Helpers::turn_on_slave(); }
$self->state($new_state);
}
return 'OK: Status applied successfully!';
}
sub check_roles($) {
my $self = shift;
foreach my $role (@{$self->roles}) {
$role->check();
}
}
sub from_config($%) {
my $self = shift;
my $config = shift;
my $host = $config->{host}->{$config->{this}};
$self->name ($config->{this});
$self->ip ($host->{ip});
$self->port ($host->{agent_port});
$self->interface ($host->{cluster_interface});
$self->mode ($host->{mode});
$self->mysql_port ($host->{mysql_port});
$self->mysql_user ($host->{agent_user});
$self->mysql_password ($host->{agent_password});
$self->writer_role ($config->{active_master_role});
$self->bin_path ($host->{bin_path});
}
1;
mysql-mmm-2.2.1/lib/Agent/Helpers.pm 0000644 0000000 0000000 00000006502 11370745632 015722 0 ustar root root package MMM::Agent::Helpers;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
our $VERSION = '0.01';
=head1 NAME
MMM::Agent::Helpers - an interface to helper programs for B
=cut
=head1 FUNCTIONS
=over 4
=item check_ip($if, $ip)
Check if the IP $ip is configured on interface $if.
Calls B.
=cut
sub check_ip($$) {
my $if = shift;
my $ip = shift;
return _execute('check_ip', "$if $ip");
}
=item configure_ip($if, $ip)
Check if the IP $ip is configured on interface $if. If not, configure it and
send arp requests to notify other hosts.
Calls B.
=cut
sub configure_ip($$) {
my $if = shift;
my $ip = shift;
return _execute('configure_ip', "$if $ip");
}
=item clear_ip($if, $ip)
Remove the IP address $ip from interface $if.
Calls B.
=cut
sub clear_ip($$) {
my $if = shift;
my $ip = shift;
return _execute('clear_ip', "$if $ip");
}
=item mysql_may_write( )
Determine wheter writes on local MySQL server are allowes.
Calls B, which reads the config file.
=cut
sub may_write() {
return _execute('mysql_may_write');
}
=item mysql_allow_write( )
Allow writes on local MySQL server. Sets global read_only to 0.
Calls B, which reads the config file.
=cut
sub allow_write() {
return _execute('mysql_allow_write');
}
=item mysql_deny_write( )
Deny writes on local MySQL server. Sets global read_only to 1.
Calls B, which reads the config file.
=cut
sub deny_write() {
return _execute('mysql_deny_write');
}
=item turn_on_slave( )
Start slave on local MySQL server.
Calls B, which reads the config file.
=cut
sub turn_on_slave() {
return _execute('turn_on_slave');
}
=item turn_off_slave( )
Stop slave on local MySQL server.
Calls B, which reads the config file.
=cut
sub turn_off_slave() {
return _execute('turn_off_slave');
}
=item sync_with_master( )
Try to sync a (soon active) master up with his peer (old active master) when the
I is moved. If peer is reachable sync with master log. If
not reachable, sync with relay log.
Calls B, which reads the config file.
=cut
sub sync_with_master() {
return _execute('sync_with_master');
}
=item set_active_master($new_master)
Try to catch up with the old master as far as possible and change the master to the new host.
(Syncs to the master log if the old master is reachable. Otherwise syncs to the relay log.)
Calls B, which reads the config file.
=cut
sub set_active_master($) {
my $new_master = shift;
return "ERROR: Unknown host $new_master" unless (defined($main::config->{host}->{$new_master}));
return _execute('set_active_master', $new_master);
}
#-------------------------------------------------------------------------------
sub _execute($$$) {
my $command = shift;
my $params = shift;
my $return_all = shift;
my $path = $main::agent->bin_path . "/agent/$command";
my $config_file = $main::agent->config_file;
$params = '' unless defined($params);
DEBUG "Executing $path $params";
my $res = `$path $config_file $params`;
unless ($return_all) {
my @lines = split /\n/, $res;
return pop(@lines);
}
return $res;
}
1;
=back
=cut
mysql-mmm-2.2.1/lib/Agent/Role.pm 0000644 0000000 0000000 00000004326 11370745632 015223 0 ustar root root package MMM::Agent::Role;
use base 'MMM::Common::Role';
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use MMM::Agent::Helpers;
our $VERSION = '0.01';
=head1 NAME
MMM::Agent::Role - role class (agent)
=cut
=head1 METHODS
=over 4
=item check()
Check (=assure) that the role is configured on the local host.
=cut
sub check($) {
my $self = shift;
my $res;
if ($self->name eq $main::agent->writer_role) {
$res = MMM::Agent::Helpers::allow_write();
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't allow writes: %s", defined($res) ? $res : 'undef');
return;
}
}
$res = MMM::Agent::Helpers::configure_ip($main::agent->interface, $self->ip);
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't configure IP '%s' on interface '%s': %s", $self->ip, $main::agent->interface, defined($res) ? $res : 'undef');
return;
}
}
=item add()
Add a role to the local host.
=cut
sub add($) {
my $self = shift;
my $res;
if ($self->name eq $main::agent->writer_role) {
$res = MMM::Agent::Helpers::sync_with_master();
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't sync with master: %s", defined($res) ? $res : 'undef');
return;
}
$res = MMM::Agent::Helpers::allow_write();
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't allow writes: %s", defined($res) ? $res : 'undef');
return;
}
}
$res = MMM::Agent::Helpers::configure_ip($main::agent->interface, $self->ip);
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't configure IP '%s' on interface '%s': %s", $self->ip, $main::agent->interface, defined($res) ? $res : 'undef');
return;
}
}
=item del()
Delete a role from the local host.
=cut
sub del($) {
my $self = shift;
my $res;
if ($self->name eq $main::agent->writer_role) {
$res = MMM::Agent::Helpers::deny_write();
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't deny writes: %s", defined($res) ? $res : 'undef');
}
}
$res = MMM::Agent::Helpers::clear_ip($main::agent->interface, $self->ip);
if (!defined($res) || $res !~ /^OK/) {
FATAL sprintf("Couldn't clear IP '%s' from interface '%s': %s", $self->ip, $main::agent->interface, defined($res) ? $res : 'undef');
}
}
1;
mysql-mmm-2.2.1/lib/Monitor/ 0000755 0000000 0000000 00000000000 11370745632 014350 5 ustar root root mysql-mmm-2.2.1/lib/Monitor/Agent.pm 0000644 0000000 0000000 00000007433 11370745632 015753 0 ustar root root package MMM::Monitor::Agent;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use MMM::Common::Socket;
use MMM::Monitor::ChecksStatus;
our $VERSION = '0.01';
use Class::Struct;
no warnings qw(Class::Struct);
struct 'MMM::Monitor::Agent' => {
host => '$',
mode => '$',
ip => '$',
port => '$',
state => '$',
roles => '@',
uptime => '$',
last_uptime => '$',
last_state_change => '$',
online_since => '$',
flapping => '$',
flapstart => '$',
flapcount => '$',
agent_down => '$'
};
sub state {
my $self = shift;
if (@_) {
my $new_state = shift;
my $old_state = $self->{'MMM::Monitor::Agent::state'};
$self->last_state_change(time()) if ($old_state ne $new_state);
$self->online_since(time()) if ($old_state ne $new_state && $new_state eq 'ONLINE');
if ($old_state ne $new_state && $main::config->{monitor}->{flap_count}) {
if ($old_state eq 'ONLINE' and $new_state ne 'ADMIN_OFFLINE') {
if (!$self->flapstart || $self->flapstart < time() - $main::config->{monitor}->{flap_duration}) {
$self->flapstart(time());
$self->flapcount(1);
}
else {
$self->{'MMM::Monitor::Agent::flapcount'}++;
if ($self->flapcount > $main::config->{monitor}->{flap_count}) {
$self->flapping(1);
$self->flapstart(0);
FATAL sprintf('Host %s is flapping!', $self->host);
}
}
}
}
$self->{'MMM::Monitor::Agent::state'} = $new_state;
warn 'Too many args to state' if @_;
}
return $self->{'MMM::Monitor::Agent::state'};
}
sub _send_command {
my $self = shift;
my $cmd = shift;
my @params = @_;
my $checks_status = MMM::Monitor::ChecksStatus->instance();
unless ($checks_status->ping($self->host)) {
return 0;
}
DEBUG sprintf("Sending command '%s(%s)' to %s (%s:%s)", $cmd, join(', ', @params), $self->host, $self->ip, $self->port);
my $socket;
CONNECT: {
$socket = MMM::Common::Socket::create_sender($self->ip, $self->port, 10);
unless ($socket && $socket->connected) {
redo CONNECT if ($!{EINTR});
return 0;
}
}
print $socket join('|', $cmd, main::MMM_PROTOCOL_VERSION, $self->host, @params), "\n";
my $res;
READ: {
$res = <$socket>;
redo READ if !$res && $!{EINTR};
}
close($socket);
unless (defined($res)) {
WARN sprintf('Received undefined answer from host %s. $!: %s', $self->host, $!);
return 0;
}
DEBUG "Received Answer: $res";
if ($res =~ /(.*)\|UP:(.*)/) {
$res = $1;
my $uptime = $2;
$self->uptime($uptime);
$self->last_uptime($uptime) if ($self->state eq 'ONLINE');
}
else {
WARN sprintf('Received bad answer \'%s\' from host %s. $!: %s', $res, $self->host, $!);
}
return $res;
}
sub _send_command_retry {
my $self = shift;
my $retries = shift;
my $cmd = shift;
my @params = @_;
my $res;
do {
$res = $self->_send_command($cmd, @params);
if ($res) { return $res; }
$retries--;
if ($retries >= 0) { DEBUG "Retrying to send command"; }
} while ($retries >= 0);
return $res;
}
sub cmd_ping($) {
my $self = shift;
my $retries = shift || 0;
return $self->_send_command_retry($retries, 'PING');
}
sub cmd_set_status($$) {
my $self = shift;
my $master = shift;
my $retries = shift || 0;
return $self->_send_command_retry($retries, 'SET_STATUS', $self->state, join(',', sort(@{$self->roles})), $master);
}
sub cmd_get_agent_status($) {
my $self = shift;
my $retries = shift || 0;
return $self->_send_command_retry($retries, 'GET_AGENT_STATUS');
}
sub cmd_get_system_status($) {
my $self = shift;
my $retries = shift || 0;
return $self->_send_command_retry($retries, 'GET_SYSTEM_STATUS');
}
sub cmd_clear_bad_roles($) {
my $self = shift;
my $retries = shift || 0;
return $self->_send_command_retry($retries, 'CLEAR_BAD_ROLES');
}
1;
mysql-mmm-2.2.1/lib/Monitor/NetworkChecker.pm 0000644 0000000 0000000 00000003362 11370745632 017630 0 ustar root root package MMM::Monitor::NetworkChecker;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use MMM::Monitor::Checker;
our $VERSION = '0.01';
=head1 NAME
MMM::Monitor::NetworkChecker - Function for checking state of network
=head1 SYNOPSIS
our $shutdown :shared = 0;
our $have_net :shared = 0;
$SIG{INT} = sub { $shutdown = 1; };
my $thread = new threads(\&MMM::Monitor::NetworkChecker::main)
=cut
sub main() {
my @ips = @{$main::config->{monitor}->{ping_ips}};
# Create checker
my $checker = new MMM::Monitor::Checker::('ping_ip');
# Perform checks until shutdown
while (!$main::shutdown) {
my $state = 0;
foreach my $ip (@ips) {
last if ($main::shutdown);
# Ping checker
$checker->spawn() unless $checker->ping();
my $res = $checker->check($ip);
if ($res =~ /^OK/) {
$state = 1;
last;
}
}
if ($main::have_net != $state) {
FATAL "Network is reachable" if ($state);
FATAL "Network is unreachable" unless ($state);
$main::have_net = $state;
}
# Sleep a while before checking every ip again
sleep($main::config->{monitor}->{ping_interval});
}
$checker->shutdown();
}
sub wait_for_network() {
my @ips = @{$main::config->{monitor}->{ping_ips}};
# Create checker
my $checker = new MMM::Monitor::Checker::('ping_ip');
while (!$main::shutdown) {
# Ping all ips
foreach my $ip (@ips) {
last if ($main::shutdown);
# Ping checker
$checker->spawn() unless $checker->ping();
my $res = $checker->check($ip);
if ($res =~ /^OK/) {
DEBUG "IP '$ip' is reachable: $res";
$checker->shutdown();
return 1;
}
}
# Sleep a while before checking every ip again
sleep($main::config->{monitor}->{ping_interval});
}
$checker->shutdown();
return 0;
}
1;
mysql-mmm-2.2.1/lib/Monitor/Checker.pm 0000644 0000000 0000000 00000012123 11370745632 016251 0 ustar root root package MMM::Monitor::Checker;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use IPC::Open2;
use MMM::Monitor::CheckResult;
our $VERSION = '0.01';
=head1 NAME
MMM::Monitor::Checker - Checker class and main function for the checker threads
=head1 SYNOPSIS
# Spawn a checker thread which will create and poll a checker and push its status to $queue
our $shutdown :shared = 0;
$SIG{INT} = sub { $shutdown = 1; };
my $queue = checker_queue(new Thread::Queue::)
my $ping_thread = new threads(\&MMM::Monitor::Checker::main, 'ping', $queue)
my $mysql_thread = new threads(\&MMM::Monitor::Checker::main, 'mysql', $queue)
...
=cut
sub main($$) {
my $check_name = shift;
my $queue = shift;
# Some shortcuts
my @checks = keys(%{$main::config->{check}});
my @hosts = keys(%{$main::config->{host}});
my $options = $main::config->{check}->{$check_name};
# Create checker
my $checker = new MMM::Monitor::Checker::($check_name);
# Initialize failure counters
my $failures = {};
foreach my $host_name (@hosts) {
$failures->{$host_name} = {
state => -1, # -1 undefined; 1 - ok; -2 untrapped error; 0 trapped error
time => 0,
}
}
# Perform checks until shutdown
while (!$main::shutdown) {
foreach my $host_name (@hosts) {
last if ($main::shutdown);
last unless ($main::have_net);
# Ping checker
$checker->spawn() unless $checker->ping();
# Check service ...
my $res = $checker->check($host_name);
# If success
if ($res =~ /^OK/) {
next if ($failures->{$host_name}->{state} == 1);
if ($failures->{$host_name}->{state} != -2) {
INFO "Check '$check_name' on '$host_name' is ok!";
$queue->enqueue(new MMM::Monitor::CheckResult::($host_name, $check_name, 1, $res));
}
$failures->{$host_name}->{time} = 0;
$failures->{$host_name}->{state} = 1;
next;
}
# If unknown
if ($res =~ /^UNKNOWN/) {
next if ($failures->{$host_name}->{state} == -3);
$failures->{$host_name}->{time} = time();
$failures->{$host_name}->{state}= -3;
WARN "Check '$check_name' on '$host_name' is in unknown state! Message: $res";
next;
}
# If failed
if ($res =~ /^ERROR/) {
last unless ($main::have_net);
next if ($failures->{$host_name}->{state} == 0);
if ($failures->{$host_name}->{state} != 0 && $failures->{$host_name}->{state} != -2) {
$failures->{$host_name}->{time} = time();
$failures->{$host_name}->{state}= -2;
}
my $failure_age = time() - $failures->{$host_name}->{time};
next if ($failure_age < $options->{trap_period});
ERROR "Check '$check_name' on '$host_name' has failed for $failure_age seconds! Message: $res";
$queue->enqueue(new MMM::Monitor::CheckResult::($host_name, $check_name, 0, $res));
$failures->{$host_name}->{state} = 0;
next;
}
}
sleep($options->{check_period});
}
$checker->shutdown();
}
=pod
# Create checker - will spawn a checker process
my $checker = new MMM::Monitor::Checker::('ping');
=cut
sub new($$) {
my $class = shift;
my $name = shift;
my $self = {};
$self->{name} = $name;
bless $self, $class;
$self->spawn();
return $self;
}
=pod
# Respawn checker if it doesn't respond
$checker->spawn() unless $checker->ping();
=cut
sub spawn($) {
my $self = shift;
my $name = $self->{name};
my $reader; # STDOUT of checker
my $writer; # STDIN of checker
INFO "Spawning checker '$name'...";
my $cluster = ($main::cluster_name ? '@' . $main::cluster_name : '');
my $pid = open2($reader, $writer, $main::config->{monitor}->{bin_path} . "/monitor/checker $cluster $name");
if (!$pid) {
LOGDIE "Can't spawn checker! Error: $!";
}
$self->{pid} = $pid;
$self->{reader} = $reader;
$self->{writer} = $writer;
}
=pod
# Shutdown checker process
$checker->shutdown();
=cut
sub shutdown($) {
my $self = shift;
my $name = $self->{name};
INFO "Shutting down checker '$name'...";
my $reader = $self->{reader};
my $writer = $self->{writer};
my $send_res = print $writer "quit\n";
my $recv_res = <$reader>;
chomp($recv_res) if defined($recv_res);
}
=pod
# Check if checker process is still alive
$checker->ping();
=cut
sub ping($) {
my $self = shift;
my $name = $self->{name};
# DEBUG "Pinging checker '$name'...";
my $reader = $self->{reader};
my $writer = $self->{writer};
my $send_res = print $writer "ping\n";
my $recv_res;
READ: {
$recv_res = <$reader>;
redo READ if !$recv_res && $!{EINTR};
}
chomp($recv_res) if defined($recv_res);
if (!$send_res || !$recv_res || !($recv_res =~ /^OK/)) {
WARN "Checker '$name' is dead!";
return 0;
}
# DEBUG "Checker '$name' is OK ($recv_res)";
return 1;
}
=pod
# Tell the checker to check host 'db2'
$checker->check('db2');
=cut
sub check($$) {
my $self = shift;
my $host = shift;
my $name = $self->{name};
my $reader = $self->{reader};
my $writer = $self->{writer};
my $send_res = print $writer "check $host\n";
my $recv_res;
READ: {
$recv_res = <$reader>;
redo READ if !$recv_res && $!{EINTR};
}
chomp($recv_res) if defined($recv_res);
return "UNKNOWN: Checker '$name' is dead!" unless ($send_res && $recv_res);
return $recv_res;
}
1;
mysql-mmm-2.2.1/lib/Monitor/ChecksStatus.pm 0000644 0000000 0000000 00000006350 11370745632 017316 0 ustar root root package MMM::Monitor::ChecksStatus;
use base 'Class::Singleton';
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use MMM::Monitor::Checker
our $VERSION = '0.01';
sub _new_instance($) {
my $class = shift;
my $data = {};
my @checks = keys(%{$main::config->{check}});
my @hosts = keys(%{$main::config->{host}});
foreach my $host_name (@hosts) {
$data->{$host_name} = {};
}
my $time = time();
# Perform initial checks
INFO 'Performing initial checks...';
foreach my $check_name (@checks) {
# Spawn checker
my $checker = new MMM::Monitor::Checker::($check_name);
# Check all hosts
foreach my $host_name (@hosts) {
DEBUG "Trying initial check '$check_name' on host '$host_name'";
my $res = $checker->check($host_name);
DEBUG "$check_name($host_name) = '$res'";
$data->{$host_name}->{$check_name} = {};
$data->{$host_name}->{$check_name}->{status} = ($res =~ /^OK/)? 1 : 0;
$data->{$host_name}->{$check_name}->{last_change} = $time;
$data->{$host_name}->{$check_name}->{message} = $res;
}
# Shutdown checker
$checker->shutdown();
}
return bless $data, $class;
}
=item handle_result(MMM::Monitor::CheckResult $result)
handle the results of a check and change state accordingly
=cut
sub handle_result($$) {
my $self = shift;
my $result = shift;
# always save the latest message, but don't override time of last change
$self->{$result->{host}}->{$result->{check}}->{message} = $result->{message};
return if ($result->{result} == $self->{$result->{host}}->{$result->{check}}->{status});
$self->{$result->{host}}->{$result->{check}}->{status} = $result->{result};
$self->{$result->{host}}->{$result->{check}}->{last_change} = time();
}
=item ping($host)
Get state of check "ping" on host $host.
=cut
sub ping($$) {
my $self = shift;
my $host = shift;
return $self->{$host}->{ping}->{status};
}
=item ping($host)
Get state of check "mysql" on host $host.
=cut
sub mysql($$) {
my $self = shift;
my $host = shift;
return $self->{$host}->{mysql}->{status};
}
=item rep_threads($host)
Get state of check "rep_threads" on host $host.
=cut
sub rep_threads($$) {
my $self = shift;
my $host = shift;
return $self->{$host}->{rep_threads}->{status};
}
=item rep_backlog($host)
Get state of check "rep_backlog" on host $host.
=cut
sub rep_backlog($$) {
my $self = shift;
my $host = shift;
return $self->{$host}->{rep_backlog}->{status};
}
=item last_change($host, [$check])
Get time of last state change
=cut
sub last_change {
my $self = shift;
my $host = shift;
my $check = shift || undef;
return $self->{$host}->{$check}->{last_change} if (defined($check));
my $time = $self->{$host}->{ping}->{last_change};
$time = $self->{$host}->{mysql}->{last_change} if ($self->{$host}->{mysql}->{last_change} > $time);
$time = $self->{$host}->{rep_threads}->{last_change} if ($self->{$host}->{rep_threads}->{last_change} > $time);
$time = $self->{$host}->{rep_backlog}->{last_change} if ($self->{$host}->{rep_backlog}->{last_change} > $time);
return $time;
}
=item message($host, $check)
Get time of last state change
=cut
sub message($$$) {
my $self = shift;
my $host = shift;
my $check = shift;
return $self->{$host}->{$check}->{message};
}
1;
mysql-mmm-2.2.1/lib/Monitor/Agents.pm 0000644 0000000 0000000 00000010036 11370745632 016127 0 ustar root root package MMM::Monitor::Agents;
use base 'Class::Singleton';
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use IO::Handle;
use File::Temp;
use File::Basename;
use MMM::Monitor::Agent;
use MMM::Monitor::Role;
=head1 NAME
MMM::Monitor::Agents - single instance class holding status information for all agent hosts
=head1 SYNOPSIS
# Get the instance
my $agents = MMM::Monitor::Agents->instance();
=cut
sub _new_instance($) {
my $class = shift;
my $data = {};
my @hosts = keys(%{$main::config->{host}});
foreach my $host (@hosts) {
$data->{$host} = new MMM::Monitor::Agent:: (
host => $host,
mode => $main::config->{host}->{$host}->{mode},
ip => $main::config->{host}->{$host}->{ip},
port => $main::config->{host}->{$host}->{agent_port},
state => 'UNKNOWN',
roles => [],
uptime => 0,
last_uptime => 0
);
}
return bless $data, $class;
}
=head1 FUNCTIONS
=over 4
=item exists($host)
Check if host $host exists.
=cut
sub exists($$) {
my $self = shift;
my $host = shift;
return defined($self->{$host});
}
=item get($host)
Get agent for host $host.
=cut
sub get($$) {
my $self = shift;
my $host = shift;
return $self->{$host};
}
=item state($host)
Get state of host $host.
=cut
sub state($$) {
my $self = shift;
my $host = shift;
LOGDIE "Can't get state of invalid host '$host'" if (!defined($self->{$host}));
return $self->{$host}->state;
}
=item online_since($host)
Get time since host $host is online.
=cut
sub online_since($$) {
my $self = shift;
my $host = shift;
LOGDIE "Can't get time since invalid host '$host' is online" if (!defined($self->{$host}));
return $self->{$host}->online_since;
}
=item set_state($host, $state)
Set state of host $host to $state.
=cut
sub set_state($$$) {
my $self = shift;
my $host = shift;
my $state = shift;
LOGDIE "Can't set state of invalid host '$host'" if (!defined($self->{$host}));
$self->{$host}->state($state);
}
=item get_status_info
Get string containing status information.
=cut
sub get_status_info($) {
my $self = shift;
my $detailed= shift || 0;
my $res = '';
my $agent_res = '';
keys (%$self); # reset iterator
foreach my $host (sort(keys(%$self))) {
my $agent = $self->{$host};
next unless $agent;
$agent_res .= "# Warning: agent on host $host is not reachable\n" if ($agent->agent_down());
$res .= sprintf(" %s(%s) %s/%s. Roles: %s\n", $host, $agent->ip, $agent->mode, $agent->state, join(', ', sort(@{$agent->roles})));
}
$res = $agent_res . $res if ($detailed);
return $res;
}
=item save_status
Save status information into status file.
=cut
sub save_status($) {
my $self = shift;
my $filename = $main::config->{monitor}->{status_path};
my ($fh, $tempname) = File::Temp::tempfile(basename($filename) . ('X' x 10), UNLINK => 0, DIR => dirname($filename));
keys (%$self); # reset iterator
while (my ($host, $agent) = each(%$self)) {
next unless $agent;
printf($fh "%s|%s|%s\n", $host, $agent->state, join(',', sort(@{$agent->roles})));
}
IO::Handle::flush($fh);
IO::Handle::sync($fh);
close($fh);
rename($tempname, $filename) || LOGDIE "Can't savely overwrite status file '$filename'!";
return;
}
=item load_status
Load status information from status file
=cut
sub load_status($) {
my $self = shift;
my $filename = $main::config->{monitor}->{status_path};
# Open status file
unless (open(STATUS, '<', $filename)) {
FATAL "Couldn't open status file '$filename': Starting up without status information.";
return;
}
while (my $line = ) {
chomp($line);
my ($host, $state, $roles) = split(/\|/, $line);
unless (defined($self->{$host})) {
WARN "Ignoring saved status information for unknown host '$host'";
next;
}
# Parse roles
my @saved_roles_str = sort(split(/\,/, $roles));
my @saved_roles = ();
foreach my $role_str (@saved_roles_str) {
my $role = MMM::Monitor::Role->from_string($role_str);
push (@saved_roles, $role) if defined($role);
}
$self->{$host}->state($state);
$self->{$host}->roles(\@saved_roles);
}
close(STATUS);
return;
}
1;
mysql-mmm-2.2.1/lib/Monitor/CheckResult.pm 0000644 0000000 0000000 00000000671 11370745632 017126 0 ustar root root package MMM::Monitor::CheckResult;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use threads;
use threads::shared;
our $VERSION = '0.01';
sub new($$$$$) {
my $class = shift;
my $host = shift;
my $check = shift;
my $result = shift;
my $message = shift;
my %self :shared;
$self{host} = $host;
$self{check} = $check;
$self{result} = $result;
$self{message} = $message;
return bless \%self, $class;
}
1;
mysql-mmm-2.2.1/lib/Monitor/Role.pm 0000644 0000000 0000000 00000000234 11370745632 015606 0 ustar root root package MMM::Monitor::Role;
use strict;
use warnings FATAL => 'all';
use MMM::Common::Role;
our $VERSION = '0.01';
our @ISA = qw(MMM::Common::Role);
1;
mysql-mmm-2.2.1/lib/Monitor/Roles.pm 0000644 0000000 0000000 00000033022 11370745632 015772 0 ustar root root package MMM::Monitor::Roles;
use base 'Class::Singleton';
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use MMM::Monitor::Agents;
use MMM::Monitor::Role;
our $VERSION = '0.01';
=head1 NAME
MMM::Monitor::Roles - holds information for all roles
=cut
sub _new_instance($) {
my $class = shift;
my $self = {};
# create list of roles - each role will be orphaned by default
foreach my $role (keys(%{$main::config->{role}})) {
my $role_info = $main::config->{role}->{$role};
my $ips = {};
foreach my $ip (@{$role_info->{ips}}) {
$ips->{$ip} = { 'assigned_to' => '' }
}
$self->{$role} = {
mode => $role_info->{mode},
hosts => $role_info->{hosts},
ips => $ips
};
if ($role_info->{mode} eq 'exclusive' && $role_info->{prefer}) {
$self->{$role}->{prefer} = $role_info->{prefer};
}
}
return bless $self, $class;
}
=head1 FUNCTIONS
=over 4
=item assign($role, $host)
Assign role $role to host $host
=cut
sub assign($$$) {
my $self = shift;
my $role = shift;
my $host = shift;
LOGDIE "Can't assign role '$role' - no host given" unless (defined($host));
# Check if the ip is still configured for this role
unless (defined($self->{$role->name}->{ips}->{$role->ip})) {
WARN sprintf("Detected configuration change: ip '%s' was removed from role '%s'", $role->ip, $role->name);
return;
}
INFO sprintf("Adding role '%s' with ip '%s' to host '%s'", $role->name, $role->ip, $host);
$self->{$role->name}->{ips}->{$role->ip}->{assigned_to} = $host;
}
=item get_role_hosts($role)
Get all hosts which may handle role $role
=cut
sub get_role_hosts($$) {
my $self = shift;
my $role = shift;
return () unless (defined($role));
my $role_info = $self->{$role};
return () unless $role_info;
return @{$role_info->{hosts}};
}
=item get_host_roles($host)
Get all roles assigned to host $host
=cut
sub get_host_roles($$) {
my $self = shift;
my $host = shift;
return () unless (defined($host));
my @roles = ();
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
foreach my $ip (keys(%{$role_info->{ips}})) {
my $ip_info = $role_info->{ips}->{$ip};
next unless ($ip_info->{assigned_to} eq $host);
push(@roles, new MMM::Monitor::Role::(name => $role, ip => $ip));
}
}
return @roles;
}
=item host_has_roles($host)
Check whether there are roles assigned to host $host
=cut
sub host_has_roles($$) {
my $self = shift;
my $host = shift;
return 0 unless (defined($host));
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
foreach my $ip (keys(%{$role_info->{ips}})) {
my $ip_info = $role_info->{ips}->{$ip};
return 1 if ($ip_info->{assigned_to} eq $host);
}
}
return 0;
}
=item count_host_roles($host)
Count all roles assigned to host $host
=cut
sub count_host_roles($$) {
my $self = shift;
my $host = shift;
return 0 unless (defined($host));
my $cnt = 0;
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
foreach my $ip (keys(%{$role_info->{ips}})) {
my $ip_info = $role_info->{ips}->{$ip};
next if ($ip_info->{assigned_to} ne $host);
$cnt++;
$cnt -= 0.5 if ($role eq $main::config->{active_master_role});
}
}
return $cnt;
}
=item get_active_master
Get the host with the active master-role
=cut
sub get_active_master($) {
my $self = shift;
my $role = $self->{$main::config->{active_master_role}};
return '' unless $role;
my @ips = keys( %{ $role->{ips} } );
return $role->{ips}->{$ips[0]}->{assigned_to};
}
=item get_passive_master
Get the passive master
=cut
sub get_passive_master($) {
my $self = shift;
my $role = $self->{$main::config->{active_master_role}};
my $active_master = $self->get_active_master();
return '' unless $role;
return '' unless $active_master;
foreach my $host ( @{ $role->{hosts} } ) {
return $host if ($host ne $active_master);
}
return '';
}
=item get_first_master
Get the first master
=cut
sub get_first_master($) {
my $self = shift;
my $role = $self->{$main::config->{active_master_role}};
return '' unless $role;
return '' unless $role->{hosts}[0];
return $role->{hosts}[0];
}
=item get_second_master
Get the second master
=cut
sub get_second_master($) {
my $self = shift;
my $role = $self->{$main::config->{active_master_role}};
return '' unless $role;
return '' unless $role->{hosts}[1];
return $role->{hosts}[1];
}
=item get_master_hosts
Get the hosts which can handle the active master-role
=cut
sub get_master_hosts($) {
my $self = shift;
my $role = $self->{$main::config->{active_master_role}};
return '' unless $role;
return $self->{$role}->{hosts};
}
=item get_exclusive_role_owner($role)
Get the host which has the exclusive role $role assigned
=cut
sub get_exclusive_role_owner($$) {
my $self = shift;
my $role = shift;
my $role_info = $self->{$role};
return '' unless $role_info;
my @ips = keys( %{ $role_info->{ips} } );
return $role_info->{ips}->{$ips[0]}->{assigned_to};
}
=item get_exclusive_role_ip($role)
Get the ip of an exclusive role $role
=cut
sub get_exclusive_role_ip($$) {
my $self = shift;
my $role = shift;
my $role_info = $self->{$role};
return undef unless $role_info;
my @ips = keys( %{ $role_info->{ips} } );
return $ips[0];
}
=item assigned_to_preferred_host($role)
Check if role is assigned to preferred host
=cut
sub assigned_to_preferred_host($$) {
my $self = shift;
my $role = shift;
my $role_info = $self->{$role};
return undef unless $role_info;
return undef unless ($role_info->{prefer});
my @ips = keys( %{ $role_info->{ips} } );
return ($role_info->{ips}->{$ips[0]}->{assigned_to} eq $role_info->{prefer});
}
=item clear_roles($host)
Remove all roles from host $host.
=cut
sub clear_roles($$) {
my $self = shift;
my $host = shift;
INFO "Removing all roles from host '$host':";
my $orphaned_master_role = 0;
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
foreach my $ip (keys(%{$role_info->{ips}})) {
my $ip_info = $role_info->{ips}->{$ip};
next unless ($ip_info->{assigned_to} eq $host);
INFO " Removed role '$role($ip)' from host '$host'";
$ip_info->{assigned_to} = '';
$orphaned_master_role = 1 if ($role eq $main::config->{active_master_role});
}
}
return $orphaned_master_role;
}
=item clear_balanced_role($host, $role)
Remove balanced role $role from host $host.
=cut
sub clear_balanced_role($$$) {
my $self = shift;
my $host = shift;
my $role = shift;
INFO "Removing balanced role $role from host '$host':";
my $role_info = $self->{$role};
return 0 unless $role_info;
my $cnt = 0;
next unless ($role_info->{mode} eq 'balanced');
foreach my $ip (keys(%{$role_info->{ips}})) {
my $ip_info = $role_info->{ips}->{$ip};
next unless ($ip_info->{assigned_to} eq $host);
$cnt++;
INFO " Removed role '$role($ip)' from host '$host'";
$ip_info->{assigned_to} = '';
}
return $cnt;
}
=item find_eligible_host($role)
find host which can take over the role $role
=cut
sub find_eligible_host($$) {
my $self = shift;
my $role = shift;
my $min_host = '';
my $min_count = 0;
my $agents = MMM::Monitor::Agents->instance();
# Maybe role has a preferred hosts
if ($self->{$role}->{prefer}) {
my $host = $self->{$role}->{prefer};
if ($agents->{$host}->state eq 'ONLINE' && !$agents->{$host}->agent_down) {
return $host;
}
}
# Use host with fewest roles
foreach my $host ( @{ $self->{$role}->{hosts} } ) {
next unless ($agents->{$host}->state eq 'ONLINE');
next if ($agents->{$host}->agent_down);
my $cnt = $self->count_host_roles($host);
next unless ($cnt < $min_count || $min_host eq '');
$min_host = $host;
$min_count = $cnt;
}
return $min_host;
}
=item find_eligible_hosts($role)
find all hosts which can take over the role $role.
=cut
sub find_eligible_hosts($$) {
my $self = shift;
my $role = shift;
my $hosts = {};
my $agents = MMM::Monitor::Agents->instance();
foreach my $host ( @{ $self->{$role}->{hosts} } ) {
next unless ($agents->{$host}->state eq 'ONLINE');
next if ($agents->{$host}->agent_down);
my $cnt = $self->count_host_roles($host);
$hosts->{$host} = $cnt;
}
return $hosts;
}
=item process_orphans
Find orphaned roles and assign them to a host if possible.
=cut
sub process_orphans($$) {
my $self = shift;
my $mode = shift;
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
next if ($mode && $role_info->{mode} ne $mode);
foreach my $ip (keys(%{$role_info->{ips}})) {
my $ip_info = $role_info->{ips}->{$ip};
next unless ($ip_info->{assigned_to} eq '');
# Find host which can take over the role - skip if none found
my $host = $self->find_eligible_host($role);
last unless ($host);
# Assign this ip to host
$ip_info->{assigned_to} = $host;
INFO "Orphaned role '$role($ip)' has been assigned to '$host'";
}
}
}
=item obey_preferences
Obey preferences by moving roles to preferred hosts
=cut
sub obey_preferences($) {
my $self = shift;
my $agents = MMM::Monitor::Agents->instance();
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
next unless ($role_info->{prefer});
my $host = $role_info->{prefer};
next unless ($agents->{$host}->state eq 'ONLINE');
next if ($agents->{$host}->agent_down);
my @ips = keys( %{ $role_info->{ips} } );
my $ip = $ips[0];
my $ip_info = $role_info->{ips}->{$ip};
my $old_host = $ip_info->{assigned_to};
next if ($old_host eq $host);
$ip_info->{assigned_to} = $host;
INFO "Moving role '$role($ip)' from host '$old_host' to preferred host '$host'";
}
}
=item get_preference_info
Get information about roles with preferred hosts
=cut
sub get_preference_info($) {
my $self = shift;
my $ret = '';
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
next unless ($role_info->{prefer});
my $host = $role_info->{prefer};
my $other_host = $self->get_exclusive_role_owner($role);
if ($host eq $other_host) {
$ret .= "# Role $role is assigned to it's preferred host $host.\n";
}
elsif($other_host ne '') {
$ret .= "# Role $role has $host configured as it's preferred host but is assigned to $other_host at the moment.\n";
}
else {
$ret .= "# Role $role has $host configured as it's preferred host.\n";
}
}
return $ret;
}
=item balance
Balance roles with mode 'balanced'
=cut
sub balance($) {
my $self = shift;
foreach my $role (keys(%$self)) {
my $role_info = $self->{$role};
next unless ($role_info->{mode} eq 'balanced');
my $hosts = $self->find_eligible_hosts($role);
next if (scalar(keys(%$hosts)) < 2);
while (1) {
my $max_host = '';
my $min_host = '';
foreach my $host (keys(%$hosts)) {
$max_host = $host if ($max_host eq '' || $hosts->{$host} > $hosts->{$max_host});
$min_host = $host if ($min_host eq '' || $hosts->{$host} < $hosts->{$min_host});
}
if ($hosts->{$max_host} - $hosts->{$min_host} <= 1) {
last;
}
$self->move_one_ip($role, $max_host, $min_host);
$hosts->{$max_host}--;
$hosts->{$min_host}++;
}
}
}
=item move_one_ip($role, $host1, $host2)
Move one IP of role $role from $host1 to $host2.
=cut
sub move_one_ip($$$$) {
my $self = shift;
my $role = shift;
my $host1 = shift;
my $host2 = shift;
foreach my $ip (keys(%{$self->{$role}->{ips}})) {
my $ip_info = $self->{$role}->{ips}->{$ip};
next unless ($ip_info->{assigned_to} eq $host1);
INFO "Moving role '$role($ip)' from host '$host1' to host '$host2'";
$ip_info->{assigned_to} = $host2;
return 1;
}
# No ip was moved
return 0;
}
=item find_by_ip($ip)
Find name of role with IP $ip.
=cut
sub find_by_ip($$) {
my $self = shift;
my $ip = shift;
foreach my $role (keys(%$self)) {
return $role if (defined($self->{$role}->{ips}->{$ip}));
}
return undef;
}
=item set_role($role, $ip, $host)
Set role $role with IP $ip to host $host.
NOTE: No checks are done. Caller should assure that:
Role is valid, IP is valid, Host is valid, Host can handle role
=cut
sub set_role($$$$) {
my $self = shift;
my $role = shift;
my $ip = shift;
my $host = shift;
$self->{$role}->{ips}->{$ip}->{assigned_to} = $host;
}
=item exists($role)
Check if role $role exists.
=cut
sub exists($$) {
my $self = shift;
my $role = shift;
return defined($self->{$role});
}
=item exists_ip($role, $ip)
Check if role $role with IP $ip exists.
=cut
sub exists_ip($$$) {
my $self = shift;
my $role = shift;
my $ip = shift;
return 0 unless defined($self->{$role});
return defined($self->{$role}->{ips}->{$ip});
}
=item is_exclusive($role)
Determine whether given role is an exclusive role.
=cut
sub is_exclusive($$) {
my $self = shift;
my $role = shift;
return 0 unless defined($self->{$role});
return ($self->{$role}->{mode} eq 'exclusive');
}
=item get_valid_hosts($role)
Get all valid hosts for role $role.
=cut
sub get_valid_hosts($$) {
my $self = shift;
my $role = shift;
return () unless defined($self->{$role});
return $self->{$role}->{hosts};
}
=item can_handle($role, $host)
Check if host $host can handle role $role.
=cut
sub can_handle($$$) {
my $self = shift;
my $role = shift;
my $host = shift;
return 0 unless defined($self->{$role});
return grep({$_ eq $host} @{$self->{$role}->{hosts}});
}
=item is_master($host)
Check if host $host can handle role $role.
=cut
sub is_master($$) {
my $self = shift;
my $host = shift;
my $role = $self->{$main::config->{active_master_role}};
return 0 unless defined($role);
return grep({$_ eq $host} @{$role->{hosts}});
}
=item is_active_master_role($role)
Check whether $role is the active master role.
=cut
sub is_active_master_role($$) {
my $self = shift;
my $role = shift;
return ($role eq $main::config->{active_master_role});
}
1;
mysql-mmm-2.2.1/lib/Monitor/Commands.pm 0000644 0000000 0000000 00000036246 11370745632 016462 0 ustar root root package MMM::Monitor::Commands;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
use List::Util qw(max);
use threads;
use threads::shared;
use Log::Log4perl::DateFormat;
use MMM::Common::Socket;
use MMM::Monitor::Agents;
use MMM::Monitor::ChecksStatus;
use MMM::Monitor::Monitor;
use MMM::Monitor::Roles;
our $VERSION = '0.01';
sub main($$) {
my $queue_in = shift;
my $queue_out = shift;
my $socket = MMM::Common::Socket::create_listener($main::config->{monitor}->{ip}, $main::config->{monitor}->{port});
while (!$main::shutdown) {
DEBUG 'Listener: Waiting for connection...';
my $client = $socket->accept();
next unless ($client);
DEBUG 'Listener: Connect!';
while (my $cmd = <$client>) {
chomp($cmd);
last if ($cmd eq 'quit');
$queue_out->enqueue($cmd);
my $res;
until ($res) {
lock($queue_in);
cond_timedwait($queue_in, time() + 1);
$res = $queue_in->dequeue_nb();
return 0 if ($main::shutdown);
}
print $client $res;
return 0 if ($main::shutdown);
}
close($client);
DEBUG 'Listener: Disconnect!';
}
}
sub ping() {
return 'OK: Pinged successfully!';
}
sub show() {
my $agents = MMM::Monitor::Agents->instance();
my $monitor = MMM::Monitor::Monitor->instance();
my $roles = MMM::Monitor::Roles->instance();
my $ret = '';
if ($monitor->is_passive) {
$ret .= "--- Monitor is in PASSIVE MODE ---\n";
$ret .= sprintf("Cause: %s\n", $monitor->passive_info);
$ret =~ s/^/# /mg;
}
$ret .= $agents->get_status_info(1);
$ret .= $roles->get_preference_info();
return $ret;
}
sub checks {
my $host = shift || 'all';
my $check = shift || 'all';
my $checks = MMM::Monitor::ChecksStatus->instance();
my $ret = '';
my $dateformat = Log::Log4perl::DateFormat->new('yyyy/MM/dd HH:mm:ss');
my @valid_checks = qw(ping mysql rep_threads rep_backlog);
return "ERROR: Unknown check '$check'!" unless ($check eq 'all' || grep(/^$check$/, @valid_checks));
if ($host ne 'all') {
return "ERROR: Unknown host name '$host'!" unless (defined($main::config->{host}->{$host}));
if ($check ne 'all') {
return sprintf("%s %s [last change: %s] %s",
$host,
$check,
$dateformat->format($checks->last_change($host, $check)),
$checks->message($host, $check)
);
}
foreach $check (@valid_checks) {
$ret .= sprintf("%s %-11s [last change: %s] %s\n",
$host,
$check,
$dateformat->format($checks->last_change($host, $check)),
$checks->message($host, $check)
);
}
return $ret;
}
my $len = 0;
foreach my $host (keys(%{$main::config->{host}})) { $len = max($len, length $host) }
if ($check ne 'all') {
foreach my $host (keys(%{$main::config->{host}})) {
$ret .= sprintf("%*s %s [last change: %s] %s\n",
$len * -1,
$host,
$check,
$dateformat->format($checks->last_change($host, $check)),
$checks->message($host, $check)
);
}
return $ret;
}
foreach my $host (keys(%{$main::config->{host}})) {
foreach $check (@valid_checks) {
$ret .= sprintf("%*s %-11s [last change: %s] %s\n",
$len * -1,
$host,
$check,
$dateformat->format($checks->last_change($host, $check)),
$checks->message($host, $check)
);
}
}
return $ret;
}
sub set_online($) {
my $host = shift;
my $agents = MMM::Monitor::Agents->instance();
return "ERROR: Unknown host name '$host'!" unless (defined($main::config->{host}->{$host}));
my $host_state = $agents->state($host);
return "OK: This host is already ONLINE. Skipping command." if ($host_state eq 'ONLINE');
unless ($host_state eq 'ADMIN_OFFLINE' || $host_state eq 'AWAITING_RECOVERY') {
return "ERROR: Host '$host' is '$host_state' at the moment. It can't be switched to ONLINE.";
}
my $checks = MMM::Monitor::ChecksStatus->instance();
if ((!$checks->ping($host) || !$checks->mysql($host))) {
return "ERROR: Checks ping and/or mysql are not ok for host '$host'. It can't be switched to ONLINE.";
}
# Check peer replication state
if ($main::config->{host}->{$host}->{peer}) {
my $peer = $main::config->{host}->{$host}->{peer};
if ($agents->state($peer) eq 'ONLINE' && (!$checks->rep_threads($peer) || !$checks->rep_backlog($peer))) {
return "ERROR: Some replication checks failed on peer '$peer'. We can't set '$host' online now. Please, wait some time.";
}
}
my $agent = MMM::Monitor::Agents->instance()->get($host);
if (!$agent->cmd_ping()) {
return "ERROR: Can't reach agent daemon on '$host'! Can't switch its state!";
}
FATAL "Admin changed state of '$host' from $host_state to ONLINE";
$agents->set_state($host, 'ONLINE');
$agent->flapping(0);
MMM::Monitor::Monitor->instance()->send_agent_status($host);
return "OK: State of '$host' changed to ONLINE. Now you can wait some time and check its new roles!";
}
sub set_offline($) {
my $host = shift;
my $agents = MMM::Monitor::Agents->instance();
return "ERROR: Unknown host name '$host'!" unless (defined($main::config->{host}->{$host}));
my $host_state = $agents->state($host);
return "OK: This host is already ADMIN_OFFLINE. Skipping command." if ($host_state eq 'ADMIN_OFFLINE');
unless ($host_state eq 'ONLINE' || $host_state eq 'REPLICATION_FAIL' || $host_state eq 'REPLICATION_DELAY') {
return "ERROR: Host '$host' is '$host_state' at the moment. It can't be switched to ADMIN_OFFLINE.";
}
my $agent = MMM::Monitor::Agents->instance()->get($host);
return "ERROR: Can't reach agent daemon on '$host'! Can't switch its state!" unless ($agent->cmd_ping());
FATAL "Admin changed state of '$host' from $host_state to ADMIN_OFFLINE";
$agents->set_state($host, 'ADMIN_OFFLINE');
MMM::Monitor::Roles->instance()->clear_roles($host);
MMM::Monitor::Monitor->instance()->send_agent_status($host);
return "OK: State of '$host' changed to ADMIN_OFFLINE. Now you can wait some time and check all roles!";
}
sub set_ip($$) {
my $ip = shift;
my $host = shift;
return "ERROR: This command is only allowed in passive mode" unless (MMM::Monitor::Monitor->instance()->is_passive);
my $agents = MMM::Monitor::Agents->instance();
my $roles = MMM::Monitor::Roles->instance();
my $role = $roles->find_by_ip($ip);
return "ERROR: Unknown ip '$ip'!" unless (defined($role));
return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host));
unless ($roles->can_handle($role, $host)) {
return "ERROR: Host '$host' can't handle role '$role'. Following hosts could: " . join(', ', @{ $roles->get_valid_hosts($role) });
}
my $host_state = $agents->state($host);
unless ($host_state eq 'ONLINE') {
return "ERROR: Host '$host' is '$host_state' at the moment. Can't move role with ip '$ip' there.";
}
FATAL "Admin set role '$role($ip)' to host '$host'";
$roles->set_role($role, $ip, $host);
# Determine all roles and propagate them to agent objects.
foreach my $one_host (@{ $roles->get_valid_hosts($role) }) {
my $agent = $agents->get($one_host);
my @agent_roles = sort($roles->get_host_roles($one_host));
$agent->roles(\@agent_roles);
}
return "OK: Set role '$role($ip)' to host '$host'.";
}
sub move_role($$) {
my $role = shift;
my $host = shift;
my $monitor = MMM::Monitor::Monitor->instance();
return "ERROR: This command is not allowed in passive mode" if ($monitor->is_passive);
my $agents = MMM::Monitor::Agents->instance();
my $roles = MMM::Monitor::Roles->instance();
return "ERROR: Unknown role name '$role'!" unless ($roles->exists($role));
return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host));
unless ($roles->is_exclusive($role)) {
$roles->clear_balanced_role($host, $role);
return "OK: Balanced role $role has been removed from host '$host'. Now you can wait some time and check new roles info!";
}
my $host_state = $agents->state($host);
return "ERROR: Can't move role to host with state $host_state." unless ($host_state eq 'ONLINE');
unless ($roles->can_handle($role, $host)) {
return "ERROR: Host '$host' can't handle role '$role'. Only following hosts could: " . join(', ', @{ $roles->get_valid_hosts($role) });
}
my $old_owner = $roles->get_exclusive_role_owner($role);
return "OK: Role is on '$host' already. Skipping command." if ($old_owner eq $host);
my $agent = MMM::Monitor::Agents->instance()->get($host);
return "ERROR: Can't reach agent daemon on '$host'! Can't move roles there!" unless ($agent->cmd_ping());
if ($monitor->is_active && $roles->assigned_to_preferred_host($role)) {
return "ERROR: Role '$role' is assigned to preferred host '$old_owner'. Can't move it!";
}
my $ip = $roles->get_exclusive_role_ip($role);
return "Error: Role $role has no IP." unless ($ip);
FATAL "Admin moved role '$role' from '$old_owner' to '$host'";
# Assign role to new host
$roles->set_role($role, $ip, $host);
# Notify old host (if is_active_master_role($role) this will make the host non writable)
$monitor->send_agent_status($old_owner);
# Notify slaves (this will make them switch the master)
$monitor->notify_slaves($host) if ($roles->is_active_master_role($role));
# Notify new host (if is_active_master_role($role) this will make the host writable)
$monitor->send_agent_status($host);
return "OK: Role '$role' has been moved from '$old_owner' to '$host'. Now you can wait some time and check new roles info!";
}
sub forced_move_role($$) {
my $role = shift;
my $host = shift;
my $monitor = MMM::Monitor::Monitor->instance();
return "ERROR: This command is not allowed in passive mode" if (MMM::Monitor::Monitor->instance()->is_passive);
my $agents = MMM::Monitor::Agents->instance();
my $roles = MMM::Monitor::Roles->instance();
my $checks = MMM::Monitor::ChecksStatus->instance();
return "ERROR: Unknown role name '$role'!" unless ($roles->exists($role));
return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host));
return "ERROR: move_role --forced may be used for the active master role only!" unless ($roles->is_active_master_role($role));
my $host_state = $agents->state($host);
unless ($host_state eq 'REPLICATION_FAIL' || $host_state eq 'REPLICATION_DELAY') {
return "ERROR: Can't force move of role to host with state $host_state.";
}
unless ($roles->can_handle($role, $host)) {
return "ERROR: Host '$host' can't handle role '$role'. Only following hosts could: " . join(', ', @{ $roles->get_valid_hosts($role) });
}
my $old_owner = $roles->get_exclusive_role_owner($role);
return "OK: Role is on '$host' already. Skipping command." if ($old_owner eq $host);
my $agent = $agents->get($host);
my $old_agent = $agents->get($old_owner);
return "ERROR: Can't reach agent daemon on '$host'! Can't move roles there!" unless ($agent->cmd_ping());
my $ip = $roles->get_exclusive_role_ip($role);
return "Error: Role $role has no IP." unless ($ip);
FATAL "Admin forced move of role '$role' from '$old_owner' to '$host'";
# Assign role to new host
$roles->set_role($role, $ip, $host);
FATAL "State of host '$host' changed from $host_state to ONLINE (because of move_role --force)";
$agent->state('ONLINE');
if (!$checks->rep_threads($old_owner)) {
FATAL "State of host '$old_owner' changed from ONLINE to REPLICATION_FAIL (because of move_role --force)";
$old_agent->state('REPLICATION_FAIL');
$roles->clear_roles($old_owner) if ($monitor->is_active);
}
elsif (!$checks->rep_backlog($old_owner)) {
FATAL "State of host '$old_owner' changed from ONLINE to REPLICATION_DELAY (because of move_role --force)";
$old_agent->state('REPLICATION_DELAY');
$roles->clear_roles($old_owner) if ($monitor->is_active);
}
# Notify old host (this will make the host non writable)
MMM::Monitor::Monitor->instance()->send_agent_status($old_owner);
# Notify slaves (this will make them switch the master)
MMM::Monitor::Monitor->instance()->notify_slaves($host);
# Notify new host (this will make the host writable)
MMM::Monitor::Monitor->instance()->send_agent_status($host);
return "OK: Role '$role' has been moved from '$old_owner' to '$host' enforcedly. Now you can wait some time and check new roles info!";
}
=item mode
Get information about current mode (active, manual or passive)
=cut
sub mode() {
my $monitor = MMM::Monitor::Monitor->instance();
return $monitor->get_mode_string();
}
=item set_active
Switch to active mode.
=cut
sub set_active() {
my $monitor = MMM::Monitor::Monitor->instance();
return 'OK: Already in active mode.' if ($monitor->is_active);
my $old_mode = $monitor->get_mode_string();
INFO "Admin changed mode from '$old_mode' to 'ACTIVE'";
if ($monitor->is_passive) {
$monitor->set_active(); # so that we can send status to agents
$monitor->cleanup_and_send_status();
$monitor->passive_info('');
}
elsif ($monitor->is_manual) {
# remove all roles from hosts which are not ONLINE
my $roles = MMM::Monitor::Roles->instance();
my $agents = MMM::Monitor::Agents->instance();
my $checks = MMM::Monitor::ChecksStatus->instance();
foreach my $host (keys(%{$main::config->{host}})) {
my $host_state = $agents->state($host);
next if ($host_state eq 'ONLINE' || $roles->get_host_roles($host) == 0);
my $agent = $agents->get($host);
$roles->clear_roles($host);
my $ret = $monitor->send_agent_status($host);
# next if ($host_state eq 'REPLICATION_FAIL');
# next if ($host_state eq 'REPLICATION_DELAY');
# NOTE host_state should never be ADMIN_OFFLINE at this point
if (!$ret) {
ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
$monitor->_kill_host($host, $checks->ping($host));
}
}
}
$monitor->set_active();
return 'OK: Switched into active mode.';
}
=item set_manual
Switch to manual mode.
=cut
sub set_manual() {
my $monitor = MMM::Monitor::Monitor->instance();
return 'OK: Already in manual mode.' if ($monitor->is_manual);
my $old_mode = $monitor->get_mode_string();
INFO "Admin changed mode from '$old_mode' to 'MANUAL'";
if ($monitor->is_passive) {
$monitor->set_manual(); # so that we can send status to agents
$monitor->cleanup_and_send_status();
$monitor->passive_info('');
}
$monitor->set_manual();
return 'OK: Switched into manual mode.';
}
=item set_passive
Switch to passive mode.
=cut
sub set_passive() {
my $monitor = MMM::Monitor::Monitor->instance();
return 'OK: Already in passive mode.' if ($monitor->is_passive);
my $old_mode = $monitor->get_mode_string();
INFO "Admin changed mode from '$old_mode' to 'PASSIVE'";
$monitor->set_passive();
$monitor->passive_info('Admin switched to passive mode.');
return 'OK: Switched into passive mode.';
}
sub help() {
return: "Valid commands are:
help - show this message
ping - ping monitor
show - show status
checks [|all [|all]] - show checks status
set_online - set host online
set_offline - set host offline
mode - print current mode.
set_active - switch into active mode.
set_manual - switch into manual mode.
set_passive - switch into passive mode.
move_role [--force] - move exclusive role to host
(Only use --force if you know what you are doing!)
set_ip - set role with ip to host
";
}
1;
mysql-mmm-2.2.1/lib/Monitor/Monitor.pm 0000644 0000000 0000000 00000075333 11370745632 016350 0 ustar root root package MMM::Monitor::Monitor;
use strict;
use warnings FATAL => 'all';
use threads;
use threads::shared;
use Algorithm::Diff;
use Data::Dumper;
use DBI;
use Errno qw(EINTR);
use Fcntl qw(F_SETFD F_GETFD FD_CLOEXEC);
use File::Temp;
use Log::Log4perl qw(:easy);
use Thread::Queue;
use MMM::Monitor::Agents;
use MMM::Monitor::Checker;
use MMM::Monitor::ChecksStatus;
use MMM::Monitor::Commands;
use MMM::Monitor::NetworkChecker;
use MMM::Monitor::Role;
use MMM::Monitor::Roles;
use MMM::Monitor::StartupStatus;
=head1 NAME
MMM::Monitor::Monitor - single instance class with monitor logic
=cut
our $VERSION = '0.01';
use constant MMM_MONITOR_MODE_PASSIVE => 0;
use constant MMM_MONITOR_MODE_ACTIVE => 1;
use constant MMM_MONITOR_MODE_MANUAL => 2;
use constant MMM_MONITOR_MODE_WAIT => 3;
use Class::Struct;
sub instance() {
return $main::monitor;
}
struct 'MMM::Monitor::Monitor' => {
checker_queue => 'Thread::Queue',
checks_status => 'MMM::Monitor::ChecksStatus',
command_queue => 'Thread::Queue',
result_queue => 'Thread::Queue',
roles => 'MMM::Monitor::Roles',
mode => '$',
passive_info => '$',
kill_host_bin => '$'
};
=head1 FUNCTIONS
=over 4
=item init
Init queues, single instance classes, ... and try to determine status.
=cut
sub init($) {
my $self = shift;
#___________________________________________________________________________
#
# Wait until network connection is available
#___________________________________________________________________________
INFO "Waiting for network connection...";
unless (MMM::Monitor::NetworkChecker->wait_for_network()) {
INFO "Received shutdown request while waiting for network connection.";
return 0;
}
INFO "Network connection is available.";
#___________________________________________________________________________
#
# Create thread queues and other stuff...
#___________________________________________________________________________
my $agents = MMM::Monitor::Agents->instance();
$self->checker_queue(new Thread::Queue::);
$self->checks_status(MMM::Monitor::ChecksStatus->instance());
$self->command_queue(new Thread::Queue::);
$self->result_queue(new Thread::Queue::);
$self->roles(MMM::Monitor::Roles->instance());
$self->passive_info('');
if ($main::config->{monitor}->{mode} eq 'active') {
$self->mode(MMM_MONITOR_MODE_ACTIVE);
}
elsif ($main::config->{monitor}->{mode} eq 'manual') {
$self->mode(MMM_MONITOR_MODE_MANUAL);
}
elsif ($main::config->{monitor}->{mode} eq 'wait') {
$self->mode(MMM_MONITOR_MODE_WAIT);
}
elsif ($main::config->{monitor}->{mode} eq 'passive') {
$self->mode(MMM_MONITOR_MODE_PASSIVE);
$self->passive_info('Configured to start up in passive mode.');
}
else {
LOGDIE "Something very, very strange just happend - dieing..."
}
#___________________________________________________________________________
#
# Check kill host binary
#___________________________________________________________________________
my $kill_host_bin = $main::config->{monitor}->{kill_host_bin};
$kill_host_bin = $main::config->{monitor}->{bin_path} . "/monitor/$kill_host_bin" unless ($kill_host_bin =~ /^\/.*/);
if (!-f $kill_host_bin) {
WARN sprintf('No binary found for killing hosts (%s).', $kill_host_bin);
}
elsif (!-x _) {
WARN sprintf('Binary for killing hosts (%s) is not executable.', $kill_host_bin);
}
else {
$self->kill_host_bin($kill_host_bin);
}
my $checks = $self->checks_status;
#___________________________________________________________________________
#
# Check replication setup of master hosts
#___________________________________________________________________________
$self->check_master_configuration();
#___________________________________________________________________________
#
# Fetch stored status, agent status and system status
#___________________________________________________________________________
$agents->load_status(); # load stored status
my $startup_status = new MMM::Monitor::StartupStatus;
my $res;
foreach my $host (keys(%{$main::config->{host}})) {
my $agent = $agents->get($host);
$startup_status->set_stored_status($host, $agent->state, $agent->roles);
#_______________________________________________________________________
#
# Get AGENT status
#_______________________________________________________________________
$res = $agent->cmd_get_agent_status(2);
if ($res =~ /^OK/) {
my ($msg, $state, $roles_str, $master) = split('\|', $res);
my @roles_str_arr = sort(split(/\,/, $roles_str));
my @roles;
foreach my $role_str (@roles_str_arr) {
my $role = MMM::Monitor::Role->from_string($role_str);
push(@roles, $role) if (defined($role));
}
$startup_status->set_agent_status($host, $state, \@roles, $master);
}
elsif ($agent->state ne 'ADMIN_OFFLINE') {
if ($checks->ping($host) && $checks->mysql($host) && !$agent->agent_down()) {
ERROR "Can't reach agent on host '$host'";
$agent->agent_down(1);
}
ERROR "The status of the agent on host '$host' could not be determined (answer was: $res).";
}
#_______________________________________________________________________
#
# Get SYSTEM status
#_______________________________________________________________________
$res = $agent->cmd_get_system_status(2);
if ($res =~ /^OK/) {
my ($msg, $writable, $roles_str, $master_ip) = split('\|', $res);
my @roles_str_arr = sort(split(/\,/, $roles_str));
my @roles;
foreach my $role_str (@roles_str_arr) {
my $role = MMM::Monitor::Role->from_string($role_str);
push(@roles, $role) if (defined($role));
}
my $master = '';
if (defined($master_ip)) {
foreach my $a_host (keys(%{$main::config->{host}})) {
$master = $a_host if ($main::config->{host}->{$a_host}->{ip} eq $master_ip);
}
}
$startup_status->set_system_status($host, $writable, \@roles, $master);
}
elsif ($agent->state ne 'ADMIN_OFFLINE') {
if ($checks->ping($host) && $checks->mysql($host) && !$agent->agent_down()) {
ERROR "Can't reach agent on host '$host'";
$agent->agent_down(1);
}
ERROR "The status of the system '$host' could not be determined (answer was: $res).";
}
}
my $conflict = $startup_status->determine_status();
DEBUG "STATE INFO\n", Data::Dumper->Dump([$startup_status], ['Startup status']);
INFO $startup_status->to_string();
foreach my $host (keys(%{$startup_status->{result}})) {
my $agent = $agents->get($host);
$agent->state($startup_status->{result}->{$host}->{state});
foreach my $role (@{$startup_status->{result}->{$host}->{roles}}) {
$self->roles->set_role($role->name, $role->ip, $host);
}
}
if ($conflict && $main::config->{monitor}->{careful_startup}) {
$self->set_passive();
$self->passive_info("Conflicting roles during startup:\n\n" . $startup_status->to_string());
}
elsif (!$self->is_passive) {
$self->cleanup_and_send_status();
}
INFO "Monitor started in active mode." if ($self->mode == MMM_MONITOR_MODE_ACTIVE);
INFO "Monitor started in manual mode." if ($self->mode == MMM_MONITOR_MODE_MANUAL);
INFO "Monitor started in wait mode." if ($self->mode == MMM_MONITOR_MODE_WAIT);
INFO "Monitor started in passive mode." if ($self->mode == MMM_MONITOR_MODE_PASSIVE);
return 1;
}
sub check_master_configuration($) {
my $self = shift;
# Get masters
my @masters = $self->roles->get_role_hosts($main::config->{active_master_role});
if (scalar(@masters) < 2) {
WARN "Only one host configured which can handle the active master role. Skipping check of master-master configuration.";
return;
}
if (scalar(@masters) > 2) {
LOGDIE "There are more than two hosts configured which can handle the active master role.";
}
# Check status of masters
my $checks = $self->checks_status;
foreach my $master (@masters) {
next if ($checks->mysql($master));
WARN "Check 'mysql' is in state 'failed' on host '$master'. Skipping check of master-master configuration.";
return;
}
# Connect to masters
my ($master1, $master2) = @masters;
my $master1_info = $main::config->{host}->{$master1};
my $master2_info = $main::config->{host}->{$master2};
my $dsn1 = sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $master1_info->{ip}, $master1_info->{mysql_port});
my $dsn2 = sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $master2_info->{ip}, $master2_info->{mysql_port});
my $eintr = EINTR;
my $dbh1;
CONNECT1: {
DEBUG "Connecting to master 1";
$dbh1 = DBI->connect($dsn1, $master1_info->{monitor_user}, $master1_info->{monitor_password}, { PrintError => 0 });
unless ($dbh1) {
redo CONNECT1 if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/);
WARN "Couldn't connect to '$master1'. Skipping check of master-master replication." . $DBI::err . " " . $DBI::errstr;
}
}
my $dbh2;
CONNECT2: {
DEBUG "Connecting to master 2";
$dbh2 = DBI->connect($dsn2, $master2_info->{monitor_user}, $master2_info->{monitor_password}, { PrintError => 0 });
unless ($dbh2) {
redo CONNECT2 if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/);
WARN "Couldn't connect to '$master2'. Skipping check of master-master replication." . $DBI::err . " " . $DBI::errstr;
}
}
# Check replication peers
my $slave_status1 = $dbh1->selectrow_hashref('SHOW SLAVE STATUS');
my $slave_status2 = $dbh2->selectrow_hashref('SHOW SLAVE STATUS');
WARN "$master1 is not replicating from $master2" if (!defined($slave_status1) || $slave_status1->{Master_Host} ne $master2_info->{ip});
WARN "$master2 is not replicating from $master1" if (!defined($slave_status2) || $slave_status2->{Master_Host} ne $master1_info->{ip});
# Check auto_increment_offset and auto_increment_increment
my ($offset1, $increment1) = $dbh1->selectrow_array('select @@auto_increment_offset, @@auto_increment_increment');
my ($offset2, $increment2) = $dbh2->selectrow_array('select @@auto_increment_offset, @@auto_increment_increment');
unless (defined($offset1) && defined($increment1)) {
WARN "Couldn't get value of auto_increment_offset/auto_increment_increment from host $master1. Skipping check of master-master replication.";
return;
}
unless (defined($offset2) && defined($increment2)) {
WARN "Couldn't get value of auto_increment_offset/auto_increment_increment from host $master2. Skipping check of master-master replication.";
return;
}
WARN "auto_increment_increment should be identical on both masters ($master1: $increment1 , $master2: $increment2)" unless ($increment1 == $increment2);
WARN "auto_increment_offset should be different on both masters ($master1: $offset1 , $master2: $offset2)" unless ($offset1 != $offset2);
WARN "$master1: auto_increment_increment ($increment1) should be >= 2" unless ($increment1 >= 2);
WARN "$master2: auto_increment_increment ($increment2) should be >= 2" unless ($increment2 >= 2);
WARN "$master1: auto_increment_offset ($offset1) should not be greater than auto_increment_increment ($increment1)" unless ($offset1 <= $increment1);
WARN "$master2: auto_increment_offset ($offset2) should not be greater than auto_increment_increment ($increment2)" unless ($offset2 <= $increment2);
}
=item main
Main thread
=cut
sub main($) {
my $self = shift;
# Delay execution so we can reap all childs before spawning the checker threads.
# This prevents a segfault if a SIGCHLD arrives during creation of a thread.
# See perl bug #60724
sleep(3);
# Spawn checker threads
my @checks = keys(%{$main::config->{check}});
my @threads;
push(@threads, new threads(\&MMM::Monitor::NetworkChecker::main));
push(@threads, new threads(\&MMM::Monitor::Commands::main, $self->result_queue, $self->command_queue));
foreach my $check_name (@checks) {
push(@threads, new threads(\&MMM::Monitor::Checker::main, $check_name, $self->checker_queue));
}
my $command_queue = $self->command_queue;
while (!$main::shutdown) {
$self->_process_check_results();
$self->_check_host_states();
$self->_process_commands();
$self->_distribute_roles();
$self->send_status_to_agents();
# sleep 3 seconds, wake up if command queue gets filled
lock($command_queue);
cond_timedwait($command_queue, time() + 3);
}
foreach my $thread (@threads) {
$thread->join();
}
}
=item _process_check_results
Process the results of the checker thread and change checks_status accordingly. Reads from check_queue.
=cut
sub _process_check_results($) {
my $self = shift;
my $cnt = 0;
while (my $result = $self->checker_queue->dequeue_nb) {
$cnt++ if $self->checks_status->handle_result($result);
}
return $cnt;
}
=item _check_host_states
Check states of hosts and change status/roles accordingly.
=cut
sub _check_host_states($) {
my $self = shift;
# Don't do anything if we have no network connection
return if (!$main::have_net);
my $checks = $self->checks_status;
my $agents = MMM::Monitor::Agents->instance();
my $active_master = $self->roles->get_active_master();
foreach my $host (keys(%{$main::config->{host}})) {
$agents->save_status() unless ($self->is_passive);
my $agent = $agents->get($host);
my $state = $agent->state;
my $ping = $checks->ping($host);
my $mysql = $checks->mysql($host);
my $rep_backlog = $checks->rep_backlog($host);
my $rep_threads = $checks->rep_threads($host);
my $peer = $main::config->{host}->{$host}->{peer};
if (!$peer && $agent->mode eq 'slave') {
$peer = $active_master
}
my $peer_state = '';
my $peer_online_since = 0;
if ($peer) {
$peer_state = $agents->state($peer);
$peer_online_since = $agents->online_since($peer);
}
# Simply skip this host. It is offlined by admin
next if ($state eq 'ADMIN_OFFLINE');
########################################################################
if ($state eq 'ONLINE') {
# ONLINE -> HARD_OFFLINE
unless ($ping && $mysql) {
FATAL sprintf("State of host '%s' changed from %s to HARD_OFFLINE (ping: %s, mysql: %s)", $host, $state, ($ping? 'OK' : 'not OK'), ($mysql? 'OK' : 'not OK'));
$agent->state('HARD_OFFLINE');
next if ($self->is_manual);
$self->roles->clear_roles($host);
if (!$self->send_agent_status($host)) {
ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
$self->_kill_host($host, $checks->ping($host));
}
next;
}
# replication failure on active master is irrelevant.
next if ($host eq $active_master);
# ignore replication failure, if peer got online recently (60 seconds, default value of master-connect-retry)
next if ($peer_state eq 'ONLINE' && $peer_online_since >= time() - 60);
# ONLINE -> REPLICATION_FAIL
if ($ping && $mysql && !$rep_threads && $peer_state eq 'ONLINE' && $checks->ping($peer) && $checks->mysql($peer)) {
FATAL "State of host '$host' changed from $state to REPLICATION_FAIL";
$agent->state('REPLICATION_FAIL');
next if ($self->is_manual);
$self->roles->clear_roles($host);
if (!$self->send_agent_status($host)) {
ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
$self->_kill_host($host, $checks->ping($host));
}
next;
}
# ONLINE -> REPLICATION_DELAY
if ($ping && $mysql && !$rep_backlog && $rep_threads && $peer_state eq 'ONLINE' && $checks->ping($peer) && $checks->mysql($peer)) {
FATAL "State of host '$host' changed from $state to REPLICATION_DELAY";
$agent->state('REPLICATION_DELAY');
next if ($self->is_manual);
$self->roles->clear_roles($host);
if (!$self->send_agent_status($host)) {
ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
$self->_kill_host($host, $checks->ping($host));
}
next;
}
next;
}
########################################################################
if ($state eq 'AWAITING_RECOVERY') {
# AWAITING_RECOVERY -> HARD_OFFLINE
unless ($ping && $mysql) {
FATAL "State of host '$host' changed from $state to HARD_OFFLINE";
$agent->state('HARD_OFFLINE');
next;
}
# AWAITING_RECOVERY -> ONLINE (if host was offline for a short period)
if ($ping && $mysql && $rep_backlog && $rep_threads) {
my $state_diff = time() - $agent->last_state_change;
if ($agent->flapping) {
my $check_state_diff = time() - $checks->last_change($host);
# set flapping host ONLINE because of auto_set_online
next unless (defined($main::config->{monitor}->{auto_set_online}) && $main::config->{monitor}->{auto_set_online} > 0);
next if ($check_state_diff < $main::config->{monitor}->{flap_duration});
FATAL sprintf(
"State of flapping host '%s' changed from %s to ONLINE because of auto_set_online and flap_duration(%d) seconds passed without another failure. It was in state AWAITING_RECOVERY for %d seconds",
$host,
$state,
$main::config->{monitor}->{flap_duration},
$state_diff
);
$agent->state('ONLINE');
$self->send_agent_status($host);
next;
}
my $uptime_diff = $agent->uptime - $agent->last_uptime;
# set ONLINE because of small downtime
if ($agent->last_uptime > 0 && $uptime_diff > 0 && $uptime_diff < 60) {
FATAL sprintf("State of host '%s' changed from %s to ONLINE because it was down for only %d seconds", $host, $state, $uptime_diff);
$agent->state('ONLINE');
$self->send_agent_status($host);
next;
}
# set ONLINE because of auto_set_online
if (defined($main::config->{monitor}->{auto_set_online}) && $main::config->{monitor}->{auto_set_online} > 0 && $main::config->{monitor}->{auto_set_online} <= $state_diff) {
FATAL sprintf("State of host '%s' changed from %s to ONLINE because of auto_set_online(%d seconds). It was in state AWAITING_RECOVERY for %d seconds", $host, $state, $main::config->{monitor}->{auto_set_online}, $state_diff);
$agent->state('ONLINE');
$self->send_agent_status($host);
next;
}
}
next;
}
########################################################################
if ($state eq 'HARD_OFFLINE') {
if ($ping && $mysql) {
# only if we have an active master or the host can't be the active master
if ($active_master ne '' || !$self->roles->can_handle($main::config->{active_master_role}, $host)) {
# HARD_OFFLINE -> REPLICATION_FAIL
if (!$rep_threads) {
FATAL "State of host '$host' changed from $state to REPLICATION_FAIL";
$agent->state('REPLICATION_FAIL');
$self->send_agent_status($host);
next;
}
# HARD_OFFLINE -> REPLICATION_DELAY
if (!$rep_backlog) {
FATAL "State of host '$host' changed from $state to REPLICATION_DELAY";
$agent->state('REPLICATION_DELAY');
$self->send_agent_status($host);
next;
}
}
# HARD_OFFLINE -> AWAITING_RECOVERY
FATAL "State of host '$host' changed from $state to AWAITING_RECOVERY";
$agent->state('AWAITING_RECOVERY');
$self->send_agent_status($host);
next;
}
}
########################################################################
if ($state eq 'REPLICATION_FAIL') {
# REPLICATION_FAIL -> REPLICATION_DELAY
if ($ping && $mysql && !$rep_backlog && $rep_threads) {
FATAL "State of host '$host' changed from $state to REPLICATION_DELAY";
$agent->state('REPLICATION_DELAY');
next;
}
}
if ($state eq 'REPLICATION_DELAY') {
# REPLICATION_DELAY -> REPLICATION_FAIL
if ($ping && $mysql && !$rep_threads) {
FATAL "State of host '$host' changed from $state to REPLICATION_FAIL";
$agent->state('REPLICATION_FAIL');
next;
}
}
########################################################################
if ($state eq 'REPLICATION_DELAY' || $state eq 'REPLICATION_FAIL') {
if ($ping && $mysql && (($rep_backlog && $rep_threads) || $peer_state ne 'ONLINE')) {
# REPLICATION_DELAY || REPLICATION_FAIL -> AWAITING_RECOVERY
if ($agent->flapping) {
FATAL "State of host '$host' changed from $state to AWAITING_RECOVERY (because it's flapping)";
$agent->state('AWAITING_RECOVERY');
$self->send_agent_status($host);
next;
}
# REPLICATION_DELAY || REPLICATION_FAIL -> ONLINE
FATAL "State of host '$host' changed from $state to ONLINE";
$agent->state('ONLINE');
$self->send_agent_status($host);
next;
}
# REPLICATION_DELAY || REPLICATION_FAIL -> HARD_OFFLINE
unless ($ping && $mysql) {
FATAL sprintf("State of host '%s' changed from %s to HARD_OFFLINE (ping: %s, mysql: %s)", $host, $state, ($ping? 'OK' : 'not OK'), ($mysql? 'OK' : 'not OK'));
$agent->state('HARD_OFFLINE');
if (!$self->send_agent_status($host)) {
ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
$self->_kill_host($host, $checks->ping($host));
}
next;
}
next;
}
}
if ($self->mode == MMM_MONITOR_MODE_WAIT) {
my $master_one = $self->roles->get_first_master();
my $master_two = $self->roles->get_second_master();
my $state_one = $agents->state($master_one);
my $state_two = $agents->state($master_two);
if ($state_one eq 'ONLINE' && $state_two eq 'ONLINE') {
INFO "Nodes $master_one and $master_two are ONLINE, switching from mode 'WAIT' to 'ACTIVE'.";
$self->set_active();
}
elsif ($main::config->{monitor}->{wait_for_other_master} > 0 && ($state_one eq 'ONLINE' || $state_two eq 'ONLINE')) {
my $living_master = $state_one eq 'ONLINE' ? $master_one : $master_two;
my $dead_master = $state_one eq 'ONLINE' ? $master_two : $master_one;
if ($main::config->{monitor}->{wait_for_other_master} <= time() - $agents->online_since($living_master)) {
$self->set_active();
WARN sprintf("Master $dead_master did not come online for %d(wait_for_other_master) seconds. Switching from mode 'WAIT' to 'ACTIVE'", $main::config->{monitor}->{wait_for_other_master});
}
}
if ($self->is_active) {
# cleanup
foreach my $host (keys(%{$main::config->{host}})) {
my $host_state = $agents->state($host);
next if ($host_state eq 'ONLINE' || $self->roles->get_host_roles($host) == 0);
my $agent = $agents->get($host);
$self->roles->clear_roles($host);
my $ret = $self->send_agent_status($host);
# next if ($host_state eq 'REPLICATION_FAIL');
# next if ($host_state eq 'REPLICATION_DELAY');
# NOTE host_state should never be ADMIN_OFFLINE at this point
if (!$ret) {
ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
$self->_kill_host($host, $checks->ping($host));
}
}
}
}
$agents->save_status() unless ($self->is_passive);
}
=item _distribute_roles
Distribute roles among the hosts.
=cut
sub _distribute_roles($) {
my $self = shift;
# Never change roles if we are in PASSIVE mode
return if ($self->is_passive);
my $old_active_master = $self->roles->get_active_master();
# Process orphaned roles
$self->roles->process_orphans('exclusive');
$self->roles->process_orphans('balanced');
# obey preferences
$self->roles->obey_preferences() if ($self->is_active);
# Balance roles
$self->roles->balance();
my $new_active_master = $self->roles->get_active_master();
# notify slaves first, if master host has changed
unless ($new_active_master eq $old_active_master) {
$self->send_agent_status($old_active_master, $new_active_master) if ($old_active_master);
$self->notify_slaves($new_active_master);
}
}
=item cleanup_and_send_status()
Send status information to all agents and clean up old roles.
=cut
sub cleanup_and_send_status($) {
my $self = shift;
my $agents = MMM::Monitor::Agents->instance();
my $roles = MMM::Monitor::Roles->instance();
my $active_master = $roles->get_active_master();
my $passive_master = $roles->get_passive_master();
# Notify passive master first
if ($passive_master ne '') {
my $host = $passive_master;
$self->send_agent_status($host);
my $agent = $agents->get($host);
$agent->cmd_clear_bad_roles(); # TODO check result
}
# Notify all slave hosts
foreach my $host (keys(%{$main::config->{host}})) {
next if ($self->roles->is_master($host));
$self->send_agent_status($host);
my $agent = $agents->get($host);
$agent->cmd_clear_bad_roles(); # TODO check result
}
# Notify active master at the end
if ($active_master ne '') {
my $host = $active_master;
$self->send_agent_status($host);
my $agent = $agents->get($host);
$agent->cmd_clear_bad_roles(); # TODO check result
}
}
=item send_status_to_agents
Send status information to all agents.
=cut
sub send_status_to_agents($) {
my $self = shift;
# Send status to all hosts
my $master = $self->roles->get_active_master();
foreach my $host (keys(%{$main::config->{host}})) {
$self->send_agent_status($host, $master);
}
}
=item notify_slaves
Notify all slave hosts (used when master changes).
=cut
sub notify_slaves($$) {
my $self = shift;
my $new_master = shift;
# Send status to all hosts with mode = 'slave'
foreach my $host (keys(%{$main::config->{host}})) {
next unless ($main::config->{host}->{$host}->{mode} eq 'slave');
$self->send_agent_status($host, $new_master);
}
}
=item send_agent_status($host[, $master])
Send status information to agent on host $host.
=cut
sub send_agent_status($$$) {
my $self = shift;
my $host = shift;
my $master = shift;
# Never send anything to agents if we are in PASSIVE mode
# Never send anything to agents if we have no network connection
return if ($self->is_passive || !$main::have_net);
# Determine active master if it was not passed
$master = $self->roles->get_active_master() unless (defined($master));
my $agent = MMM::Monitor::Agents->instance()->get($host);
# Determine and set roles
my @roles = sort($self->roles->get_host_roles($host));
$agent->roles(\@roles);
# Finally send command
my $ret = $agent->cmd_set_status($master);
unless ($ret) {
# If ping is down, nothing will be send to agent. So this doesn't indicate that the agent is down.
my $checks = $self->checks_status;
if ($checks->ping($host) && !$agent->agent_down()) {
FATAL "Can't reach agent on host '$host'";
$agent->agent_down(1);
}
}
elsif ($agent->agent_down) {
FATAL "Agent on host '$host' is reachable again";
$agent->agent_down(0);
}
return $ret;
}
=item _kill_host
Process commands received from the command thread.
=cut
sub _kill_host($$$) {
my $self = shift;
my $host = shift;
my $ping = shift;
if (!defined($self->kill_host_bin)) {
FATAL sprintf("Could not kill host '%s' - there may be some duplicate ips now! (There's no binary configured for killing hosts.)", $host);
return;
}
# Executing the kill_host_bin and capturing its output _and_ return code is a bit complicated.
# We can't use backticks - $? (also called $CHILD_ERROR) will always be undefined because
# mmm_mond installs a custom signal handler for SIGCHLD.
# So we use "system" instead of backticks and redirect the output to a temporary file.
# To prevent race conditions we use tempfile instead of tmpname, clear the close-on-exec flag
# and redirect the output of system to '/dev/fd/' . fileno(fh).
my $fh = File::Temp::tempfile();
my $flags = fcntl($fh, F_GETFD, 0);
$flags &= ~FD_CLOEXEC;
fcntl($fh, F_SETFD, $flags);
my $command = sprintf("%s %s %s", $self->kill_host_bin, $host, $ping);
INFO sprintf("Killing host using command '%s'", $command);
my $ret = system($command . sprintf(' >/dev/fd/%s 2>&1', fileno($fh)));
# signal information in the lower 8 bits, exit code above that
$ret = $ret >> 8;
my $output = '';
seek($fh, 0, 0);
local $/;
$output = <$fh>;
close $fh;
if ($ret == 0) {
INFO sprintf("Output of kill host command was: %s", $output) if ($output ne "");
return;
}
FATAL sprintf("Could not kill host '%s' - there may be some duplicate ips now! kill_host binary exited with '%s'. Output was: %s ", $host, $ret, $output);
return;
}
=item _process_commands
Process commands received from the command thread.
=cut
sub _process_commands($) {
my $self = shift;
# Handle all queued commands
while (my $cmdline = $self->command_queue->dequeue_nb) {
# Parse command
my @args = split(/\s+/, $cmdline);
my $command = shift @args;
my $arg_cnt = scalar(@args);
my $res;
# Execute command
if ($command eq 'help' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::help(); }
elsif ($command eq 'ping' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::ping(); }
elsif ($command eq 'show' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::show(); }
elsif ($command eq 'checks' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::checks(); }
elsif ($command eq 'checks' && $arg_cnt == 1) { $res = MMM::Monitor::Commands::checks($args[0]); }
elsif ($command eq 'checks' && $arg_cnt == 2) { $res = MMM::Monitor::Commands::checks($args[0], $args[1]); }
elsif ($command eq 'mode' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::mode(); }
elsif ($command eq 'set_active' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::set_active(); }
elsif ($command eq 'set_passive' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::set_passive(); }
elsif ($command eq 'set_manual' && $arg_cnt == 0) { $res = MMM::Monitor::Commands::set_manual(); }
elsif ($command eq 'set_online' && $arg_cnt == 1) { $res = MMM::Monitor::Commands::set_online ($args[0]); }
elsif ($command eq 'set_offline' && $arg_cnt == 1) { $res = MMM::Monitor::Commands::set_offline($args[0]); }
elsif ($command eq 'move_role' && $arg_cnt == 2) { $res = MMM::Monitor::Commands::move_role($args[0], $args[1]); }
elsif ($command eq 'move_role' && $arg_cnt == 3 && $args[0] eq "--force") {
$res = MMM::Monitor::Commands::forced_move_role($args[1], $args[2]);
}
elsif ($command eq 'set_ip' && $arg_cnt == 2) { $res = MMM::Monitor::Commands::set_ip($args[0], $args[1]); }
else { $res = "Invalid command '$cmdline'\n\n" . MMM::Monitor::Commands::help(); }
# Enqueue result
$self->result_queue->enqueue($res);
}
}
=item is_active()
Check if monitor is in active mode
=cut
sub is_active($$) {
my $self = shift;
return ($self->mode == MMM_MONITOR_MODE_ACTIVE);
}
=item is_manual()
Check if monitor is in manual mode
=cut
sub is_manual($$) {
my $self = shift;
return ($self->mode == MMM_MONITOR_MODE_MANUAL || $self->mode == MMM_MONITOR_MODE_WAIT);
}
=item is_passive()
Check if monitor is in passive mode
=cut
sub is_passive($$) {
my $self = shift;
return ($self->mode == MMM_MONITOR_MODE_PASSIVE);
}
=item set_active()
Set mode to active
=cut
sub set_active($$) {
my $self = shift;
$self->mode(MMM_MONITOR_MODE_ACTIVE);
}
=item set_manual()
Set mode to manual
=cut
sub set_manual($$) {
my $self = shift;
$self->mode(MMM_MONITOR_MODE_MANUAL);
}
=item set_passive()
Set mode to passive
=cut
sub set_passive($$) {
my $self = shift;
$self->mode(MMM_MONITOR_MODE_PASSIVE);
}
=item get_mode_string()
Get string representation of current mode
=cut
sub get_mode_string($) {
my $self = shift;
return 'ACTIVE' if ($self->mode == MMM_MONITOR_MODE_ACTIVE);
return 'MANUAL' if ($self->mode == MMM_MONITOR_MODE_MANUAL);
return 'WAIT' if ($self->mode == MMM_MONITOR_MODE_WAIT);
return 'PASSIVE' if ($self->mode == MMM_MONITOR_MODE_PASSIVE);
return 'UNKNOWN'; # should never happen
}
1;
mysql-mmm-2.2.1/lib/Monitor/Checker/ 0000755 0000000 0000000 00000000000 11370745632 015714 5 ustar root root mysql-mmm-2.2.1/lib/Monitor/Checker/Checks.pm 0000644 0000000 0000000 00000017677 11370745632 017474 0 ustar root root package MMM::Monitor::Checker::Checks;
use strict;
use warnings FATAL => 'all';
use English qw(OSNAME EFFECTIVE_USER_ID);
use DBI;
use Net::Ping;
use POSIX ':signal_h';
our $VERSION = '0.01';
=head1 NAME
MMM::Monitor::Checker::Checks - functions for the B helper program B
=cut
my $fping_path;
=head1 FUNCTIONS
=over 4
=item ping($timeout, $host)
Check if the host $host is reachable.
=cut
sub ping($$) {
my $timeout = shift;
my $host = shift;
my $ip = $main::config->{host}->{$host}->{ip};
return "ERROR: Invalid host '$host'" unless ($ip);
# if super user, use Net::Ping - it's faster
if ($EFFECTIVE_USER_ID == 0) {
my $p = Net::Ping->new('icmp');
$p->hires();
if ($p->ping($ip, 0.5)) {
return 'OK';
}
return "ERROR: Could not ping $ip";
}
# Find appropriate fping version
_determine_fping_path() unless defined($fping_path);
unless (defined($fping_path)) {
return "ERROR: fping is not functional - please, install your own version of fping on this server!";
}
my $res = `$fping_path -q -u -t 500 -C 1 $ip 2>&1`;
return "ERROR: fping could not reach $ip" if ($res =~ /$ip.*\-$/);
return 'OK';
}
=item ping_ip($timeout, $ip)
Check if the IP $ip is reachable.
=cut
sub ping_ip($$) {
my $timeout = shift;
my $ip = shift;
# if super user, use Net::Ping - it's faster
if ($EFFECTIVE_USER_ID == 0) {
my $p = Net::Ping->new('icmp');
$p->hires();
if ($p->ping($ip, 0.5)) {
return 'OK';
}
return "ERROR: Could not ping $ip";
}
# Find appropriate fping version
_determine_fping_path() unless defined($fping_path);
unless (defined($fping_path)) {
return "ERROR: fping is not functional - please, install your own version of fping on this server!";
}
my $res = `$fping_path -q -u -t 500 -C 1 $ip 2>&1`;
return "ERROR: fping could not reach $ip" if ($res =~ /$ip.*\-$/);
return 'OK';
}
=item mysql($timeout, $host)
Check if the mysql server on host $host is reachable.
=cut
sub mysql($$) {
my $timeout = shift;
my $host = shift;
my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($host);
return "ERROR: Invalid host '$host'" unless ($peer_host);
my $mask = POSIX::SigSet->new( SIGALRM );
my $action = POSIX::SigAction->new(
sub { die 'TIMEOUT'; },
$mask,
);
my $oldaction = POSIX::SigAction->new();
sigaction( SIGALRM, $action, $oldaction );
my $res = eval {
alarm($timeout + 1);
# connect to server
my $dsn = "DBI:mysql:host=$peer_host;port=$peer_port;mysql_connect_timeout=$timeout";
my $dbh = DBI->connect($dsn, $peer_user, $peer_password, { PrintError => 0 });
unless ($dbh) {
alarm(0);
# We don't want to trigger any action because of a simple 'too many connections' error
return "UNKNOWN: Too many connections! " . $DBI::errstr if ($DBI::err == 1040);
return "ERROR: Connect error (host = $peer_host:$peer_port, user = $peer_user)! " . $DBI::errstr;
}
# Check server (simple)
my $res = $dbh->do('SELECT NOW()');
unless ($res) {
alarm(0);
return 'ERROR: SQL Query Error: ' . $dbh->errstr;
}
alarm(1);
$dbh->disconnect();
$dbh = undef;
alarm(0);
return 0;
};
alarm(0);
return $res if ($res);
return 'ERROR: Timeout' if ($@ =~ /^TIMEOUT/);
return "UNKNOWN: Error occurred: $@" if $@;
return 'OK';
}
=item rep_backlog($timeout, $host)
Check the replication backlog on host $host.
=cut
sub rep_backlog($$) {
my $timeout = shift;
my $host = shift;
my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($host);
return "ERROR: Invalid host '$host'" unless ($peer_host);
my $mask = POSIX::SigSet->new( SIGALRM );
my $action = POSIX::SigAction->new(
sub { die 'TIMEOUT'; },
$mask,
);
my $oldaction = POSIX::SigAction->new();
sigaction( SIGALRM, $action, $oldaction );
my $res = eval {
alarm($timeout + 1);
# connect to server
my $dsn = "DBI:mysql:host=$peer_host;port=$peer_port;mysql_connect_timeout=$timeout";
my $dbh = DBI->connect($dsn, $peer_user, $peer_password, { PrintError => 0 });
unless ($dbh) {
alarm(0);
return "UNKNOWN: Connect error (host = $peer_host:$peer_port, user = $peer_user)! " . $DBI::errstr;
}
# Check server (replication backlog)
my $sth = $dbh->prepare('SHOW SLAVE STATUS');
my $res = $sth->execute;
if ($dbh->err) {
alarm(1);
my $ret = 'UNKNOWN: Unknown state. Execute error: ' . $dbh->errstr;
$ret = "ERROR: The monitor user '$peer_user' doesn't have the required REPLICATION CLIENT privilege! " . $dbh->errstr if ($dbh->err == 1227);
$sth->finish();
$dbh->disconnect();
$dbh = undef;
alarm(0);
return $ret;
}
unless ($res) {
alarm(1);
$sth->finish();
$dbh->disconnect();
$dbh = undef;
alarm(0);
return 'ERROR: Replication is not running';
}
my $status = $sth->fetchrow_hashref;
alarm(1);
$sth->finish();
$dbh->disconnect();
$dbh = undef;
alarm(0);
return 'ERROR: Replication is not set up' unless defined($status);
# Check backlog size
my $backlog = $status->{Seconds_Behind_Master};
$backlog = 0 unless ($backlog);
return 'OK: Backlog is null' if ($backlog == 0);
return 'ERROR: Backlog is too big' if ($backlog > $main::check->{max_backlog});
return 0;
};
alarm(0);
return $res if ($res);
return 'ERROR: Timeout' if ($@ =~ /^TIMEOUT/);
return "UNKNOWN: Error occurred: $@" if $@;
return 'OK';
}
=item rep_threads($timeout, $host)
Check if the mysql slave threads on host $host are running.
=cut
sub rep_threads($$) {
my $timeout = shift;
my $host = shift;
my ($peer_host, $peer_port, $peer_user, $peer_password) = _get_connection_info($host);
return "ERROR: Invalid host '$host'" unless ($peer_host);
my $mask = POSIX::SigSet->new( SIGALRM );
my $action = POSIX::SigAction->new(
sub { die 'TIMEOUT'; },
$mask,
);
my $oldaction = POSIX::SigAction->new();
sigaction( SIGALRM, $action, $oldaction );
my $res = eval {
alarm($timeout + 1);
# connect to server
my $dsn = "DBI:mysql:host=$peer_host;port=$peer_port;mysql_connect_timeout=$timeout";
my $dbh = DBI->connect($dsn, $peer_user, $peer_password, { PrintError => 0 });
return "UNKNOWN: Connect error (host = $peer_host:$peer_port, user = $peer_user)! " . $DBI::errstr unless ($dbh);
# Check server (replication backlog)
my $sth = $dbh->prepare('SHOW SLAVE STATUS');
my $res = $sth->execute;
if ($dbh->err) {
alarm(1);
my $ret = 'UNKNOWN: Unknown state. Execute error: ' . $dbh->errstr;
$ret = "ERROR: The monitor user '$peer_user' doesn't have the required REPLICATION CLIENT privilege! " . $dbh->errstr if ($dbh->err == 1227);
$sth->finish();
$dbh->disconnect();
$dbh = undef;
alarm(0);
return $ret;
}
unless ($res) {
alarm(1);
$sth->finish();
$dbh->disconnect();
$dbh = undef;
alarm(0);
return 'ERROR: Replication is not running';
}
my $status = $sth->fetchrow_hashref;
alarm(1);
$sth->finish();
$dbh->disconnect();
$dbh = undef;
alarm(0);
return 'ERROR: Replication is not set up' unless defined($status);
# Check peer replication state
if ($status->{Slave_IO_Running} eq 'No' || $status->{Slave_SQL_Running} eq 'No') {
return 'ERROR: Replication is broken';
}
return 0;
};
alarm(0);
return $res if ($res);
return 'ERROR: Timeout' if ($@ =~ /^TIMEOUT/);
return "UNKNOWN: Error occurred: $@" if $@;
return 'OK';
}
=item _get_connection_info($host)
Get connection info for host $host.
=cut
sub _get_connection_info($) {
my $host = shift;
return (
$main::config->{host}->{$host}->{ip},
$main::config->{host}->{$host}->{mysql_port},
$main::config->{host}->{$host}->{monitor_user},
$main::config->{host}->{$host}->{monitor_password}
);
}
=item _determine_fping_path()
Determine path of fping binary
=cut
sub _determine_fping_path() {
$fping_path = _determine_path('fping');
}
sub _determine_path($) {
my $program = shift;
my @paths = qw(/usr/sbin /sbin /usr/bin /bin);
foreach my $path (@paths) {
my $fullpath = "$path/$program";
if (-f $fullpath && -x $fullpath) {
return $fullpath;
}
}
return undef;
}
1;
mysql-mmm-2.2.1/lib/Monitor/StartupStatus.pm 0000644 0000000 0000000 00000022314 11370745632 017556 0 ustar root root package MMM::Monitor::StartupStatus;
use strict;
use warnings FATAL => 'all';
use List::Util qw(max);
use Log::Log4perl qw(:easy);
use MMM::Common::Role;
use MMM::Monitor::Role;
use MMM::Monitor::Roles;
our $VERSION = '0.01';
=head1 NAME
MMM::Monitor::StartupStatus - holds information about agent/system/stored status during startup
=cut
sub new($) {
my $class = shift;
my $self = {
roles => {},
hosts => {},
result=> {}
};
return bless $self, $class;
}
=head1 FUNCTIONS
=over 4
=item set_agent_status($host, $state, $roles, $master)
Set agent status
=cut
sub set_agent_status($$\@$) {
my $self = shift;
my $host = shift;
my $state = shift;
my $roles = shift;
my $master = shift;
$self->{hosts}->{$host} = {} unless (defined($self->{hosts}->{$host}));
$self->{hosts}->{$host}->{agent} = {
state => $state,
master => $master
};
foreach my $role (@{$roles}) {
unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) {
WARN "Detected change in role definitions: Role '$role' was removed.";
next;
}
unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) {
WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
next;
}
my $role_str = $role->to_string();
$self->{roles}->{$role_str} = {} unless (defined($self->{roles}->{$role_str}));
$self->{roles}->{$role_str}->{$host} = {} unless (defined($self->{roles}->{$role_str}->{$host}));
$self->{roles}->{$role_str}->{$host}->{agent} = 1;
}
}
=item set_stored_status($host, $state, $roles)
Set stored status
=cut
sub set_stored_status($$\@$) {
my $self = shift;
my $host = shift;
my $state = shift;
my $roles = shift;
$self->{hosts}->{$host} = {} unless (defined($self->{hosts}->{$host}));
$self->{hosts}->{$host}->{stored} = {
state => $state,
};
foreach my $role (@{$roles}) {
unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) {
WARN "Detected change in role definitions: Role '$role' was removed.";
next;
}
unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) {
WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
next;
}
my $role_str = $role->to_string();
$self->{roles}->{$role_str} = {} unless (defined($self->{roles}->{$role_str}));
$self->{roles}->{$role_str}->{$host} = {} unless (defined($self->{roles}->{$role_str}->{$host}));
$self->{roles}->{$role_str}->{$host}->{stored} = 1;
}
}
=item set_system_status($host, $writable, $roles, $master)
Set system status
=cut
sub set_system_status($$\@$) {
my $self = shift;
my $host = shift;
my $writable= shift;
my $roles = shift;
my $master = shift;
$self->{hosts}->{$host} = {} unless (defined($self->{hosts}->{$host}));
$self->{hosts}->{$host}->{system} = {
writable=> $writable,
master => $master
};
foreach my $role (@{$roles}) {
unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) {
WARN "Detected change in role definitions: Role '$role' was removed.";
next;
}
unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) {
WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
next;
}
my $role_str = $role->to_string();
$self->{roles}->{$role_str} = {} unless (defined($self->{roles}->{$role_str}));
$self->{roles}->{$role_str}->{$host} = {} unless (defined($self->{roles}->{$role_str}->{$host}));
$self->{roles}->{$role_str}->{$host}->{system} = 1;
}
}
sub determine_status() {
my $self = shift;
my $roles = MMM::Monitor::Roles->instance();
my $is_manual = MMM::Monitor::Monitor->instance()->is_manual();
my $conflict = 0;
foreach my $host (keys(%{$main::config->{host}})) {
# Figure out host state
my $stored_state = 'UNKNOWN';
my $agent_state = 'UNKNOWN';
my $state;
$stored_state = $self->{hosts}->{$host}->{stored}->{state} if (defined($self->{hosts}->{$host}->{stored}->{state}));
$agent_state = $self->{hosts}->{$host}->{agent}->{state} if (defined($self->{hosts}->{$host}->{agent}->{state} ));
if ( $stored_state eq 'ADMIN_OFFLINE' || $agent_state eq 'ADMIN_OFFLINE' ) { $state = 'ADMIN_OFFLINE'; }
elsif ($stored_state eq 'HARD_OFFLINE' || $agent_state eq 'HARD_OFFLINE' ) { $state = 'HARD_OFFLINE'; }
elsif ($stored_state eq 'REPLICATION_FAIL' || $agent_state eq 'REPLICATION_FAIL' ) { $state = 'REPLICATION_FAIL'; }
elsif ($stored_state eq 'REPLICATION_DELAY' || $agent_state eq 'REPLICATION_DELAY') { $state = 'REPLICATION_DELAY'; }
elsif ($stored_state eq 'ONLINE' || $agent_state eq 'ONLINE' ) { $state = 'ONLINE'; }
else { $state = 'AWAITING_RECOVERY'; }
$self->{result}->{$host} = { state => $state, roles => [] };
}
foreach my $role_str (keys(%{$self->{roles}})) {
my $role = MMM::Monitor::Role->from_string($role_str);
next unless(defined($role));
if ($roles->is_active_master_role($role->name)) {
# active master role
my $max = 0;
my $target = undef;
my $system_cnt = 0;
foreach my $host (keys(%{$self->{roles}->{$role_str}})) {
my $votes = 0;
my $info = $self->{roles}->{$role_str}->{$host};
my $host_info = $self->{hosts}->{$host};
# host is writable
$votes += 4 if (defined($host_info->{system}->{writable}) && $host_info->{system}->{writable});
# IP is configured
if (defined($info->{system})) {
$votes += 2;
$system_cnt++;
}
$votes += 1 if (defined($info->{stored}));
$votes += 1 if (defined($info->{agent}));
foreach my $slave_host (keys(%{$self->{hosts}})) {
my $slave_info = $self->{hosts}->{$slave_host};
next if MMM::Monitor::Roles->instance()->is_master($slave_host);
$votes++ if (defined($slave_info->{system}->{master}) && $slave_info->{system}->{master} eq $host);
}
my $state = $self->{result}->{$host}->{state};
$votes = 0 if ($state eq 'ADMIN_OFFLINE');
$votes = 0 if ($state eq 'HARD_OFFLINE' && !$is_manual);
if ($votes > $max) {
$target = $host;
$max = $votes;
}
}
if ($system_cnt > 1) {
WARN "Role '$role_str' was configured on $system_cnt hosts during monitor startup.";
$conflict = 1;
}
if (defined($target)) {
push (@{$self->{result}->{$target}->{roles}}, $role);
my $state = $self->{result}->{$target}->{state};
$self->{result}->{$target}->{state} = 'ONLINE' if (!$is_manual || $state eq 'REPLICATION_FAIL' || $state eq 'REPLICATION_DELAY');
}
next;
}
# Handle non-writer roles
my $max = 0;
my $target = undef;
my $system_cnt = 0;
foreach my $host (keys(%{$self->{roles}->{$role_str}})) {
my $votes = 0;
my $info = $self->{roles}->{$role_str}->{$host};
# IP is configured
if (defined($info->{system})) {
$votes += 4;
$system_cnt++;
}
$votes += 2 if (defined($info->{stored}));
$votes += 1 if (defined($info->{agent}));
my $state = $self->{result}->{$host}->{state};
if ($state eq 'ADMIN_OFFLINE' || (!$is_manual && $state ne 'ONLINE' && $state ne 'AWAITING_RECOVERY')) {
$votes = 0;
}
if ($votes > $max) {
$target = $host;
$max = $votes;
}
}
if ($system_cnt > 1) {
WARN "Role '$role_str' was configured on $system_cnt hosts during monitor startup.";
}
if (defined($target)) {
push (@{$self->{result}->{$target}->{roles}}, $role);
$self->{result}->{$target}->{state} = 'ONLINE' if ($self->{result}->{$target}->{state} eq 'AWAITING_RECOVERY');
}
}
return $conflict;
}
sub to_string($) {
my $self = shift;
my $ret = "Startup status:\n";
$ret .= "\nRoles:\n";
my $role_len = 4; # "Role"
my $host_len = 6; # "Master"
foreach my $role (keys(%{$main::config->{role}})) { $role_len = max($role_len, length $role) }
foreach my $host (keys(%{$main::config->{host}})) { $host_len = max($host_len, length $host) }
$role_len += 17; # "(999.999.999.999)"
$ret .= sprintf(" %-*s %-*s %-6s %-6s %-5s\n", $role_len, 'Role', $host_len, 'Host', 'Stored', 'System', 'Agent');
foreach my $role (keys(%{$self->{roles}})) {
foreach my $host (keys(%{$self->{roles}->{$role}})) {
my $info = $self->{roles}->{$role}->{$host};
$ret .= sprintf(" %-*s %-*s %-6s %-6s %-5s\n", $role_len, $role, $host_len, $host,
defined($info->{stored}) ? 'Yes' : '-',
defined($info->{system}) ? 'Yes' : '-',
defined($info->{agent}) ? 'Yes' : '-'
);
}
}
$ret .= "\nHosts:\n";
$ret .= sprintf(" %-*s %-*s %-8s %-16s %-16s\n", $host_len, 'Host', $host_len, 'Master', 'Writable', 'Stored state', 'Agent state');
foreach my $host (keys(%{$self->{hosts}})) {
my $info = $self->{hosts}->{$host};
my $is_master = MMM::Monitor::Roles->instance()->is_master($host);
$ret .= sprintf(" %-*s %-*s %-8s %-16s %-16s\n", $host_len, $host, $host_len,
$is_master ? '-' : (defined($info->{system}->{master}) ? $info->{system}->{master} : '?'),
defined($info->{system}->{writable}) ? ($info->{system}->{writable} ? 'Yes' : 'No') : '?',
defined($info->{stored}->{state}) ? $info->{stored}->{state} : '?',
defined($info->{agent}->{state}) ? $info->{agent}->{state} : '?',
);
}
return $ret;
}
1;
mysql-mmm-2.2.1/lib/Tools/ 0000755 0000000 0000000 00000000000 11370745632 014021 5 ustar root root mysql-mmm-2.2.1/lib/Tools/Snapshot/ 0000755 0000000 0000000 00000000000 11370745632 015620 5 ustar root root mysql-mmm-2.2.1/lib/Tools/Snapshot/MySQL.pm 0000644 0000000 0000000 00000003426 11370745632 017130 0 ustar root root package MMM::Tools::Snapshot::MySQL;
use strict;
use warnings FATAL => 'all';
use Data::Dumper;
use DBI;
use Log::Log4perl qw(:easy);
sub lock_tables($) {
my $dbh = shift;
INFO 'Locking tables...';
my $res = $dbh->do('FLUSH TABLES WITH READ LOCK');
INFO "Result: '$res'";
system('sync');
sleep(1);
system('sync');
return $res;
}
sub unlock_tables($) {
my $dbh = shift;
return $dbh->do('UNLOCK TABLES');
}
sub get_pos_info($$) {
my $dbh = shift;
my $pos_info = shift;
# Get master status info
my $res = $dbh->selectrow_hashref('SHOW MASTER STATUS');
return "ERROR: Can't get master status information! Error: " . $dbh->errstr unless ($res);
$pos_info->{master} = $res;
# Get slave status info
$res = $dbh->selectrow_hashref('SHOW SLAVE STATUS');
return "ERROR: Can't get slave status information! Error: " . $dbh->errstr if (defined($dbh->err));
$res = {} unless ($res);
$pos_info->{slave} = $res;
return 'OK: Got status info!';
}
sub save_pos_info($$) {
my $pos_info = shift;
my $file = shift;
open(POSFILE, '>', $file) || return "ERROR: Can't create pos file: $!";
print POSFILE Dumper($pos_info);
close(POSFILE);
return 'OK: Saved position info';
}
=item _get_connection_info($host)
Get connection info for host $host
=cut
sub get_connection_info($) {
my $host = shift;
# TODO maybe check $host
return (
$main::config->{host}->{$host}->{ip},
$main::config->{host}->{$host}->{mysql_port},
$main::config->{host}->{$host}->{tools_user},
$main::config->{host}->{$host}->{tools_password}
);
}
sub connect($) {
my $host_name = shift;
my ($host, $port, $user, $password) = get_connection_info($host_name);
my $dsn = "DBI:mysql:host=$host;port=$port;mysql_connect_timeout=3";
return DBI->connect($dsn, $user, $password, { PrintError => 0 });
}
1;
mysql-mmm-2.2.1/lib/Tools/Snapshot/LVM.pm 0000644 0000000 0000000 00000002640 11370745632 016616 0 ustar root root package MMM::Tools::Snapshot::LVM;
use strict;
use warnings FATAL => 'all';
use Log::Log4perl qw(:easy);
sub create() {
my $this = $main::config->{this};
my $host = $main::config->{host}->{$this};
my @command = (
$host->{lvm_bin_lvcreate},
'--snapshot',
'--size', $host->{lvm_snapshot_size},
'--name', 'mmm_snapshot',
join('/', '/dev' , $host->{lvm_volume_group}, $host->{lvm_logical_volume})
);
my $lvm_res = system(@command);
INFO "lvcreate output: '$lvm_res'";
return "ERROR: Can't create snapshot: $!" if ($lvm_res);
my $mount_opts = $host->{lvm_mount_opts};
$mount_opts = '-o rw' unless ($mount_opts);
my $res = system(sprintf('mount %s /dev/%s/mmm_snapshot %s', $mount_opts, $host->{lvm_volume_group}, $host->{lvm_mount_dir}));
return "ERROR: Can't mount snapshot: $!" if ($res);
return 'OK: Snapshot createg!';
}
sub remove() {
my $this = $main::config->{this};
my $host = $main::config->{host}->{$this};
if (!$host) {
return "ERROR: Invalid 'this' value: '$this'!";
}
# Unmount snapshot
my $res = system('umount', $host->{lvm_mount_dir});
return "ERROR: Can't umount snapshot: $!" if ($res);
my @command = (
$host->{lvm_bin_lvremove},
'-f',
join('/', '/dev', $host->{lvm_volume_group}, 'mmm_snapshot')
);
my $lvm_res = system(@command);
INFO "lvremove output: '$lvm_res'";
return "ERROR: Can't remove snapshot: $!" if ($lvm_res);
return 'OK: Snapshot removed!';
}
1;
mysql-mmm-2.2.1/lib/Tools/MySQL.pm 0000644 0000000 0000000 00000011510 11370745632 015322 0 ustar root root package MMM::Tools::MySQL;
use strict;
use warnings FATAL => 'all';
use DBI;
use Log::Log4perl qw(:easy);
our $VERSION = '0.01';
=head1 NAME
MMM::Tools::MySQL - MySQL related functions for the mmm-tools.
=cut
=over 4
=item is_running([$pid])
Check if mysqld is running
=cut
sub is_running {
my $pid = shift;
unless ($pid) {
my $pidfile = $main::config->{host}->{$main::config->{this}}->{mysql_pidfile};
return 0 unless (-f $pidfile);
open(PID, $pidfile) || LOGDIE "ERROR: Can't read MySQL pid file '$pidfile'";
chomp($pid = );
close(PID);
}
my $cnt = kill(0, $pid);
return $pid if ($cnt);
return 0;
}
=item start
Start mysqld
=cut
sub start() {
my $rcscript = $main::config->{host}->{$main::config->{this}}->{mysql_rcscript};
my $pid = is_running();
if ($pid) {
ERROR "Error: Local MySQL server is running with pid $pid";
return 0;
}
INFO 'MySQL is not running. Going to start it...';
my $res = system($rcscript, 'start');
if ($res) {
ERROR "ERROR: Can't start local MySQL server!";
return 0;
}
INFO 'MySQL has been started!';
return 1;
}
=item stop
Stop mysqld
=cut
sub stop() {
my $rcscript = $main::config->{host}->{$main::config->{this}}->{mysql_rcscript};
my $pid = is_running();
unless ($pid) {
WARN 'MySQL is not running now, skipping shutdown ...';
return 1;
}
my $res = system($rcscript, 'stop');
if ($res) {
ERROR "ERROR: Can't stop local MySQL server!";
return 0;
}
my $wait = 15;
DEBUG ("Waiting MySQL process with $pid to shutdown: ");
while ($wait--) {
$pid = is_running($pid);
last if ($pid == 0);
DEBUG '.';
sleep(1);
}
if ($pid != 0) {
ERROR "ERROR: MySQL is running with PID $pid after shutdown request!";
return 0;
}
INFO 'MySQL has been stopped!';
return 1;
}
=item change_master_to(named_params)
Required params
master_host
master_port
master_user
master_pass
Optional params
master_log
master_pos
=cut
sub change_master_to {
my $args = shift;
$args->{host} ||= $main::config->{this};
LOGDIE 'Bad call of change_master_to()' unless (
defined($args->{master_host}) && defined($args->{master_port})
&& defined($args->{master_user}) && defined($args->{master_pass})
);
INFO "Changing master of host $args->{host} to $args->{master_host} ...";
# Get connection information
my ($host, $port, $user, $password) = _get_connection_info($args->{host});
unless (defined($host)) {
ERROR "No connection info for host '$args->{host}'";
return 0;
}
# Connect to server
my $dbh = _connect($host, $port, $user, $password);
unless ($dbh) {
ERROR "Can't connect to MySQL (host = $host:$port, user = $user)!";
return 0;
}
my $res;
# Stop slave
$res = $dbh->do('STOP SLAVE');
unless ($res) {
ERROR 'SQL Query Error: ', $dbh->errstr;
return 0;
}
# Force deletion of obsolete master.info, relay-log.info and relay logs.
$res = $dbh->do('RESET SLAVE');
unless ($res) {
ERROR 'SQL Query Error: ', $dbh->errstr;
return 0;
}
# Change master
my $sql = sprintf(
"CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%s, MASTER_USER='%s', MASTER_PASSWORD='%s'",
$args->{master_host}, $args->{master_port}, $args->{master_user}, $args->{master_pass}
);
if ($args->{master_log} && $args->{master_pos}) {
$sql .= sprintf(", MASTER_LOG_FILE='%s', MASTER_LOG_POS=%s", $args->{master_log}, $args->{master_pos});
}
$res = $dbh->do($sql);
unless ($res) {
ERROR 'SQL Query Error: ', $dbh->errstr;
return 0;
}
# Start slave
$res = $dbh->do('START SLAVE');
unless ($res) {
ERROR 'SQL Query Error: ', $dbh->errstr;
return 0;
}
# Disconnect
$dbh->disconnect;
INFO "Successfully changed master.";
return 1;
}
=item _get_connection_info($host)
Get connection info for host $host
=cut
sub _get_connection_info($) {
my $host = shift;
# TODO maybe check $host
return (
$main::config->{host}->{$host}->{ip},
$main::config->{host}->{$host}->{mysql_port},
$main::config->{host}->{$host}->{tools_user},
$main::config->{host}->{$host}->{tools_password}
);
}
sub _connect($$$$) {
my ($host, $port, $user, $password) = @_;
my $dsn = "DBI:mysql:host=$host;port=$port;mysql_connect_timeout=3";
return DBI->connect($dsn, $user, $password, { PrintError => 0 });
}
sub get_master_host($) {
my $host_name = shift;
# Get connection information
my ($host, $port, $user, $password) = _get_connection_info($host_name);
unless (defined($host)) {
ERROR "No connection info for host '$host_name'";
return undef;
}
# Connect to server
my $dbh = _connect($host, $port, $user, $password);
unless ($dbh) {
ERROR "Can't connect to MySQL (host = $host:$port, user = $user)!";
return undef;
}
# Get slave status
my $res = $dbh->selectrow_hashref('SHOW SLAVE STATUS');
return "ERROR: Can't get slave status for host '$host_name'! Error: " . $dbh->errstr unless ($res);
# Disconnect
$dbh->disconnect();
return $res->{Master_Host};
}
1;
mysql-mmm-2.2.1/lib/Tools/Tools.pm 0000644 0000000 0000000 00000027012 11370745632 015461 0 ustar root root package MMM::Tools::Tools;
use strict;
use warnings FATAL => 'all';
use English qw(FORMAT_NAME);
use Path::Class qw(dir);
use Log::Log4perl qw(:easy);
our $VERSION = '0.01';
=head1 NAME
MMM::Tools::Tools - functions for the mmm-tools.
=cut
=over 4
=item check_backup_destination($path, [$should_be_empty])
Check backup destination
=cut
sub check_backup_destination {
my $dir = shift;
my $should_be_empty = shift || 0;
INFO "Checking local destination directory '$dir'...";
system("mkdir -p $dir");
unless (-d $dir && -x _ && -r _ && -w _) {
ERROR "Destination dir '$dir' has invalid permissions (should be readable/writeable/executable)!";
return 0;
}
if ($should_be_empty && scalar(glob("$dir/*"))) {
ERROR "Destination dir '$dir' is not empty!";
return 0;
}
INFO 'Directory is ok';
return 1;
}
sub check_restore_source($) {
my $dir = shift;
INFO "Checking restore source directory '$dir'...";
unless (-d $dir && -x _ && -r _) {
ERROR "Source dir '$dir' has invalid permissions (should be readable/executable)!";
return 0;
}
unless (scalar(glob("$dir/*"))) {
ERROR "Source dir '$dir' is empty!";
return 0;
}
unless (-d "$dir/_mmm" && -r _ && -x _) {
ERROR "Source dir doesn't contain _mmm sub-directory!";
return 0;
}
unless (-f "$dir/_mmm/status.txt" && -r _) {
ERROR "$dir/_mmm/status.txt doesn't exist or isn't readable!";
return 0;
}
unless (-f "$dir/_mmm/copy_method.txt" && -r _) {
ERROR "$dir/_mmm/copy_method.txt doesn't exist or isn't readable!";
return 0;
}
INFO 'Directory is ok';
return 1;
}
sub check_restore_destination($) {
my $dir = shift;
INFO "Checking destination directory '$dir'...";
system("mkdir -p $dir");
unless (-d $dir && -x _ && -r _ && -w _) {
ERROR "Destination dir '$dir' has invalid permissions (should be readable/writeable/executable)!";
return 0;
}
INFO 'Directory is ok';
return 1;
}
=item check_ssh_connection($host)
Check SSH connection to host $host.
=cut
sub check_ssh_connection($) {
my $host = shift;
my $ssh_host = $main::config->{host}->{$host}->{ssh_user} . '@' . $main::config->{host}->{$host}->{ip};
my $ssh_port = $main::config->{host}->{$host}->{ssh_port};
my $ssh_params = $main::config->{host}->{$host}->{ssh_parameters};
my $check_cmd = "ssh $ssh_params -p $ssh_port $ssh_host date";
INFO "Verifying ssh connection to remote host '$ssh_host' (command: $check_cmd)...";
my $res = system($check_cmd);
if ($res) {
ERROR "Can't execute remote commands on host '$host' ($ssh_host): $!";
return 0;
}
INFO "OK: SSH connection works fine!";
return 1;
}
=item execute_remote($host, $program)
Execute command $program on remote host $host.
=cut
sub execute_remote($$) {
my $host = shift;
my $program = shift;
my $ssh_host = $main::config->{host}->{$host}->{ssh_user} . '@' . $main::config->{host}->{$host}->{ip};
my $ssh_port = $main::config->{host}->{$host}->{ssh_port};
my $ssh_params = $main::config->{host}->{$host}->{ssh_parameters};
my $command = $main::config->{host}->{$host}->{bin_path} . '/tools/' . $program;
DEBUG "Executing $program on host '$host'...";
INFO "ssh $ssh_params -p $ssh_port $ssh_host $command";
chomp(my $res = `ssh $ssh_params -p $ssh_port $ssh_host $command`);
print "$res\n";
my @res_lines = split(/\n/, $res);
my $last_line = pop(@res_lines);
unless ($last_line =~ /^OK/) {
ERROR $res;
return 0;
}
return 1;
}
=item create_remote_snapshot($host)
Create snapshot on host $host.
=cut
sub create_remote_snapshot($) {
my $host = shift;
return execute_remote($host, 'create_snapshot');
}
=item remove_remote_snapshot($host)
Remove snapshot on host $host.
=cut
sub remove_remote_snapshot($) {
my $host = shift;
return execute_remote($host, 'remove_snapshot');
}
sub save_copy_method($$) {
my $dir = shift;
my $copy_method = shift;
$dir .= '/_mmm';
# Check config option
if (! -d $dir) {
ERROR "Directory _mmm doesn't exist!";
return 0;
}
unless (open(F, ">$dir/copy_method.txt")) {
ERROR "I/O Error while saving copy method!";
return 0;
}
print F $copy_method;
close(F);
DEBUG "Saved copy method";
return 1;
}
sub load_status($) {
my $dir = shift;
my $status;
DEBUG 'Loading status info...';
my $status_file = $dir . '/_mmm/status.txt';
unless (-f $status_file && -r _) {
ERROR "Status file '$status_file' doesn't exist or isn't readable!";
return undef;
}
my $status_data = `cat $status_file`;
my $VAR1;
eval($status_data);
if ($@) {
ERROR "Can't parse status info: $@";
return undef;
}
$status = $VAR1;
my $method_file = $dir . '/_mmm/copy_method.txt';
chomp(my $copy_method = `cat $method_file`);
$status->{copy_method} = $copy_method;
return \%{$status};
}
sub copy_clone_dirs($$$) {
my $host = shift;
my $copy_method = shift;
my $dest_dir = shift;
my @clone_dirs = @{$main::config->{clone_dirs}};
if ($main::config->{copy_method}->{$copy_method}->{single_run}) {
return copy_from_remote_single_run($host, $copy_method, $dest_dir, \@clone_dirs);
}
foreach my $sub_dir (@clone_dirs) {
return 0 unless (copy_from_remote($host, $copy_method, $dest_dir, $sub_dir));
}
return 1;
}
sub copy_from_remote($$$$) {
my $host = shift;
my $copy_method = shift;
my $dest_dir = shift;
my $sub_dir = shift;
my $host_info = $main::config->{host}->{$host};
my $ssh_host = $host_info->{ssh_user} . '@' . $host_info->{ip};
INFO "Copying '$sub_dir' from snapshot on host '$host' with copy method '$copy_method'";
my $command = $main::config->{copy_method}->{$copy_method}->{backup_command};
my $dir = dir('/', $sub_dir);
unless ($dir->parent() eq '/') {
$dest_dir .= '/' . $dir->parent();
system("mkdir -p $dest_dir");
}
$command =~ s/%SSH_USER%/$host_info->{ssh_user}/ig;
$command =~ s/%IP%/$host_info->{ip}/ig;
$command =~ s/%SNAPSHOT%/$host_info->{lvm_mount_dir}/ig;
$command =~ s/%DEST_DIR%/$dest_dir/ig;
$command =~ s/%BACKUP_DIR%/$dest_dir/ig;
$command =~ s/%CLONE_DIR%/$sub_dir/ig;
INFO "Executing command $command";
if (system($command)) {
ERROR "Can't copy $sub_dir: $!";
return 0;
}
INFO "Copied directory $sub_dir!";
return 1;
}
sub copy_from_remote_single_run($$$$) {
my $host = shift;
my $copy_method = shift;
my $dest_dir = shift;
my $clone_dirs = shift;
my $host_info = $main::config->{host}->{$host};
my $ssh_host = $host_info->{ssh_user} . '@' . $host_info->{ip};
INFO "Copying files from snapshot on host '$host' with copy method '$copy_method'";
my $command = $main::config->{copy_method}->{$copy_method}->{backup_command};
$command =~ s/%SSH_USER%/$host_info->{ssh_user}/ig;
$command =~ s/%IP%/$host_info->{ip}/ig;
$command =~ s/%SNAPSHOT%/$host_info->{lvm_mount_dir}/ig;
$command =~ s/%DEST_DIR%/$dest_dir/ig;
$command =~ s/%BACKUP_DIR%/$dest_dir/ig;
if ($command =~ /!(.*)!/) {
my $sub_tmpl = $1;
my $sub_cmd = "";
for my $sub_dir (@$clone_dirs) {
my $partial = $sub_tmpl;
$partial =~ s/%CLONE_DIR%/$sub_dir/ig;
$sub_cmd .= " $partial";
}
$command =~ s/!.*!/$sub_cmd/;
}
INFO "Executing command $command";
if (system($command)) {
# TODO New config entry "check command"?
# system("rdiff-backup --check-destination-dir '$config->{dest_dir}'");
ERROR "Can't copy from remote host: $!";
return 0;
}
INFO sprintf("Copied directories '%s' from host '$host'!", join("', '", sort(@$clone_dirs)));
return 1;
}
=item restore($copy_method, $src_dir, $dest_dir)
restore non-incremental backup
=cut
sub restore($$$) {
my $copy_method = shift;
my $src_dir = shift;
my $dest_dir = shift;
# TODO check copy method
if ($main::config->{copy_method}->{$copy_method}->{incremental}) {
ERROR 'The backup directory contains an incremental backup! Use --version option to restore a specific version.';
return 0;
}
my $command = $main::config->{copy_method}->{$copy_method}->{restore_command};
$command =~ s/%SRC_DIR%/$src_dir/ig;
$command =~ s/%BACKUP_DIR%/$src_dir/ig;
$command =~ s/%DEST_DIR%/$dest_dir/ig;
$command =~ s/%DATA_DIR%/$dest_dir/ig;
INFO "Executing command $command";
if (system($command)) {
ERROR "Can't restore data: $!";
return 0;
}
INFO "Restored backup from '$src_dir' to '$dest_dir'";
return 1;
}
=item restore_incremental($copy_method, $src_dir, $dest_dir, $version)
restore incremental backup
=cut
sub restore_incremental($$$$) {
my $copy_method = shift;
my $src_dir = shift;
my $dest_dir = shift;
my $version = shift;
# TODO check copy method
unless ($main::config->{copy_method}->{$copy_method}->{incremental}) {
ERROR 'The backup directory contains an non-incremental backup!';
return 0;
}
my $command = $main::config->{copy_method}->{$copy_method}->{restore_command};
$command =~ s/%SRC_DIR%/$src_dir/ig;
$command =~ s/%BACKUP_DIR%/$src_dir/ig;
$command =~ s/%DEST_DIR%/$dest_dir/ig;
$command =~ s/%DATA_DIR%/$dest_dir/ig;
$command =~ s/%VERSION%/$version/ig;
INFO "Executing command $command";
if (system($command)) {
ERROR "Can't restore data: $!";
return 0;
}
INFO "Restored backup version '$version' from '$src_dir' to '$dest_dir'";
return 1;
}
=item list_increments($backup_dir, $copy_method)
list available backup increments
=cut
sub list_increments($$) {
my $backup_dir = shift;
my $copy_method = shift;
my $command = $main::config->{copy_method}->{$copy_method}->{incremental_command};
unless ($main::config->{copy_method}->{$copy_method}->{incremental}) {
ERROR 'Invalid backup directory for incremental operations';
exit(0);
}
$command =~ s/%BACKUP_DIR%/$backup_dir/ig;
# List versions
my $res = open(COMMAND, "$command|");
unless ($res) {
LogError("Can't read version info from backup!");
exit(1);
}
my $line;
if ($command =~ /rdiff-backup/ && $command =~ 'parsable-output') {
# Beautify rdiff-backup output
print "Following backup versions are available:\n";
print " Version | Date\n";
print "-------------|---------------------------\n";
my $timestamp;
format VERSION_LINE =
@>>>>>>>>>> | @<<<<<<<<<<<<<<<<<<<<<<<<
$timestamp, scalar(localtime($timestamp))
.
$FORMAT_NAME = 'VERSION_LINE';
while ($line = ) {
chomp $line;
($timestamp,) = split(/\s+/, $line);
write;
}
}
else {
while ($line = ) {
print $line;
}
}
close(COMMAND);
}
=item cleanup($status, $dir)
clean up restore directory
=cut
sub cleanup($$$) {
my $status = shift;
my $dir = shift;
my $clone_dirs = shift;
INFO 'Cleaning dump from master.info and binary logs...';
my $master_log = $status->{master}->{File};
unless ($master_log =~ /^(.*)\.(\d+)$/) {
ERROR "Unknown master binary log file name format '$master_log'!";
return 0;
}
INFO "Deleting master binary logs: $1.*";
system("find $dir -name '$1.*' | xargs rm -vf");
if ($status->{slave} && $status->{slave}->{Relay_Log_File}) {
my $slave_log = $status->{slave}->{Relay_Log_File};
unless ($slave_log =~ /^(.*)\.(\d+)$/) {
ERROR "Unknown relay binary log file name format '$slave_log'!";
return 0;
}
INFO "Deleting relay binary logs: $1.*";
system("find $dir -name '$1.*' | xargs rm -vf");
}
INFO 'Deleting .info and .pid files...';
system("find $dir -name master.info | xargs rm -vf");
system("find $dir -name relay-log.info | xargs rm -vf");
system("find $dir -name '*.pid' | xargs rm -vf");
INFO 'Changing permissions on mysql data dir...';
foreach my $sub_dir (@$clone_dirs) {
system("chown -R mysql:mysql $dir/$sub_dir");
}
return 1;
}
=item get_host_by_ip($ip)
get hostname of host with ip $ip.
=cut
sub get_host_by_ip($) {
my $ip = shift;
foreach my $host (keys(%{$main::config->{host}})) {
return $host if ($main::config->{host}->{$host}->{ip} eq $ip);
}
return '';
}
1;
mysql-mmm-2.2.1/sbin/ 0000755 0000000 0000000 00000000000 11370745632 013106 5 ustar root root mysql-mmm-2.2.1/sbin/mmm_agentd 0000755 0000000 0000000 00000004452 11370745632 015151 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use English qw( PROGRAM_NAME );
use File::Basename;
use Proc::Daemon;
use Log::Log4perl qw(:easy);
Log::Log4perl->easy_init($INFO);
# Define version and protocol version
use constant MMM_VERSION => '2.2.1';
use constant MMM_PROTOCOL_VERSION => 1;
# Include parts of the system
use MMM::Common::Angel;
use MMM::Common::Config;
use MMM::Common::Log;
use MMM::Common::PidFile;
use MMM::Agent::Agent;
# Maybe we were just asked for our version
if (scalar(@ARGV) && $ARGV[0] eq "--version") {
printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION;
exit(0);
}
chdir('/');
umask(0022);
our $cluster_name = '';
my $postfix = '';
if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) {
shift(@ARGV);
$cluster_name = $1;
$postfix = "_$cluster_name";
$PROGRAM_NAME = basename($PROGRAM_NAME) . '-' . $cluster_name;
}
else {
$PROGRAM_NAME = basename($PROGRAM_NAME);
}
MMM::Common::Log::init("mmm_agent_log$postfix.conf", "mmm_agentd$postfix");
# Read configuration
our $config = new MMM::Common::Config::;
$config->read("mmm_agent$postfix");
$config->check('AGENT');
my $debug = $config->{debug};
MMM::Common::Log::debug() if ($debug);
our $agent = new MMM::Agent::Agent::(
protocol_version => 1,
active_master => '',
state => 'UNKNOWN',
config_file => "mmm_agent$postfix"
);
$agent->from_config($config);
my $pidfilename = $config->{host}->{ $config->{this} }->{pid_path};
my $pidfile = new MMM::Common::PidFile:: $pidfilename;
# Check pid file
LOGDIE "Can't run second copy of ", $PROGRAM_NAME if ($pidfile->is_running());
WARN "Unclean start - found stale pid file!" if ($pidfile->exists());
unless ($debug) {
# Go to background
Proc::Daemon::Init();
# Set umask again
umask(0022);
# Init logging again to re-open fds
MMM::Common::Log::init("mmm_agent_log$postfix.conf", "mmm_agentd$postfix");
}
# Init angel magic, which will restart us if we die unexpected
MMM::Common::Angel::Init($pidfile);
# Shutdown flag
our $shutdown = 0;
# Set signal handlers
$SIG{INT} = \&ShutdownHandler;
$SIG{TERM} = \&ShutdownHandler;
$SIG{QUIT} = \&ShutdownHandler;
$agent->main();
INFO 'END';
exit(0);
#-----------------------------------------------------------------
sub ShutdownHandler() {
INFO "Signal received: exiting...";
$shutdown = 1;
}
mysql-mmm-2.2.1/sbin/mmm_control 0000755 0000000 0000000 00000002150 11370745632 015360 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use English qw( PROGRAM_NAME );
use File::Basename;
use Log::Log4perl qw(:easy);
Log::Log4perl->easy_init($INFO);
# Define version
use constant MMM_VERSION => '2.2.1';
# Include parts of the system
use MMM::Common::Config;
use MMM::Common::Socket;
# Maybe we were just asked for our version
if (scalar(@ARGV) && $ARGV[0] eq "--version") {
printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION;
exit(0);
}
my $postfix = '';
if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) {
shift(@ARGV);
$postfix = "_$1";
}
# Read configuration
our $config = new MMM::Common::Config::;
$config->read("mmm_mon$postfix");
$config->check('CONTROL');
die "See '$0 help' for usage information" if (scalar(@ARGV) < 1);
my $socket = MMM::Common::Socket::create_sender($config->{monitor}->{ip}, $config->{monitor}->{port}, 10);
unless ($socket && $socket->connected) {
print "ERROR: Can't connect to monitor daemon!\n";
exit(1);
}
print $socket join(' ', @ARGV), "\nquit\n";
my $res = '';
my $line;
$res .= $line while ($line = <$socket>);
print $res, "\n";
exit(0);
mysql-mmm-2.2.1/sbin/mmm_restore 0000755 0000000 0000000 00000014300 11370745632 015363 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use English qw( PROGRAM_NAME FORMAT_NAME );
use File::Basename;
use Log::Log4perl qw(:easy);
use Getopt::Long;
Log::Log4perl->easy_init( { level => $INFO, layout => '%p: %m%n' });
# Define version
use constant MMM_VERSION => '2.2.1';
# Include parts of the system
use MMM::Common::Config;
use MMM::Tools::Tools;
use MMM::Tools::MySQL;
# Maybe we were just asked for our version
if (scalar(@ARGV) == 1 && $ARGV[0] eq "--version") {
printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION;
exit(0);
}
my @RESTORE_MODES = qw(
data-only
single-single
slave-single
master-single
master-slave
slave-slave
);
my $config_file = '';
my $src_dir = '';
my $dest_dir = '';
my $restore_mode= '';
my $version = '';
my $skip_mysqld = 0;
my $dry_run = 0;
print_usage() unless (
GetOptions(
'config=s' => \$config_file,
'src-dir=s' => \$src_dir,
'dest-dir=s' => \$dest_dir,
'mode=s' => \$restore_mode,
'version=s' => \$version,
'dry-run' => \$dry_run
)
);
# Set defaults (part 1)
$config_file = 'mmm_tools' if ($config_file eq '');
$restore_mode = 'single-single' if ($restore_mode eq '');
$skip_mysqld = 1 if ($restore_mode eq 'data-only');
# Read configuration
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('TOOLS');
my $host = $config->{host}->{$config->{this}};
# Set defaults (part 2)
$src_dir = $host->{backup_dir} if ($src_dir eq '');
$dest_dir = $host->{restore_dir} if ($dest_dir eq '');
# Check params
print_usage("Invalid restore mode '$restore_mode'") unless (grep(/^$restore_mode$/, @RESTORE_MODES));
print_usage("Invalid backup directory '$src_dir'") unless (MMM::Tools::Tools::check_restore_source($src_dir));
if ($version eq 'list') {
# Print list of increments
die unless (MMM::Tools::Tools::list_increments($src_dir, 'rdiff'));
exit 0;
}
# Load information from status file
my $status = MMM::Tools::Tools::load_status($src_dir);
die unless (defined($status));
# Check copy method of backup
unless (defined($config->{copy_method}->{$status->{copy_method}})) {
ERROR "Backup was created with unknown copy-method '$status->{copy_method}' - can't restore it";
die;
}
# Check if backup is incremental and we were called with version
if ($config->{copy_method}->{$status->{copy_method}}->{incremental} && $version eq '') {
ERROR "'$src_dir' contains an incremental backup, but no version was specified";
MMM::Tools::Tools::list_increments($src_dir, 'rdiff');
die;
}
# Determine replication peer
$restore_mode =~ /(\w+)\-(\w+)/;
my $src_mode = $1;
my $dest_mode = $2;
my $peer_host = $status->{host};
my $peer_host_info;
if ($src_mode eq 'slave') {
$peer_host = MMM::Tools::Tools::get_host_by_ip($status->{slave}->{'Master_Host'});
}
unless ($dest_mode eq 'single' || $skip_mysqld) {
unless (defined($config->{host}->{$peer_host})) {
LOGDIE "Unknown peer host '$peer_host'";
}
$peer_host_info = $config->{host}->{$peer_host};
}
# Print info
my $label;
my $value;
format RESTORE_INFO =
@<<<<<<<<<<<<<<<<<<<<<: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
$label, $value
.
$FORMAT_NAME = 'RESTORE_INFO';
$label = 'Source dir';
$value = $src_dir;
write;
$label = 'Destination dir';
$value = $dest_dir;
write;
$label = 'Dirs to restore';
$value = join(', ', sort(@{$config->{clone_dirs}}));
write;
$label = 'Restore mode';
$value = $restore_mode;
write;
if ($version ne '') {
$label = 'Incremental version';
$value = $version;
write;
}
if (defined($peer_host_info)) {
$label = 'Replication peer';
$value = $peer_host;
write;
}
$label = 'Skip mysqld operations';
$value = $skip_mysqld ? 'yes' : 'no';
write;
$label = 'Dry run';
$value = $dry_run ? 'yes' : 'no';
write;
# Stop here if this is a dry-run
exit 0 if ($dry_run);
unless ($skip_mysqld) {
# Stop MySQL server
die unless (MMM::Tools::MySQL::stop());
}
# Check/Create destination directory
die unless (MMM::Tools::Tools::check_restore_destination($dest_dir));
# Restore backup
if ($version eq '') {
die unless (MMM::Tools::Tools::restore($status->{copy_method}, $src_dir, $dest_dir));
}
else {
die unless (MMM::Tools::Tools::restore_incremental($status->{copy_method}, $src_dir, $dest_dir, $version));
}
# Cleanup
die unless (MMM::Tools::Tools::cleanup($status, $dest_dir, $config->{clone_dirs}));
unless ($skip_mysqld) {
# Start MySQL server
die unless (MMM::Tools::MySQL::start());
# Setup replication
if ($dest_mode eq 'single') {
INFO "Skipping replication setup because destination configuration is '$dest_mode'";
}
else {
my %dest_replication_info = (
host => $config->{this},
master_host => $peer_host_info->{ip},
master_port => $peer_host_info->{mysql_port},
master_user => $peer_host_info->{replication_user},
master_pass => $peer_host_info->{replication_password},
);
if ($src_mode eq 'master' && $dest_mode eq 'slave') {
$dest_replication_info{master_log} = $status->{master}->{'File'};
$dest_replication_info{master_pos} = $status->{master}->{'Position'};
die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info);
}
elsif ($src_mode eq 'slave' && $dest_mode eq 'slave') {
$dest_replication_info{master_log} = $status->{slave}->{'Relay_Master_Log_File'};
$dest_replication_info{master_pos} = $status->{slave}->{'Exec_Master_Log_Pos'};
die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info);
}
else {
WARN "Bad restore mode '$src_mode-$dest_mode', skipping replication setup";
}
}
}
INFO 'Restore operation finished!';
exit 0;
sub print_usage {
my $msg = shift;
print "$msg\n\n" if ($msg);
print "Usage: $0 [--config ] [--mode ] [--version ] [--src-dir ] [--dest-dir ] [--dry-run]\n";
print "Where:\n";
print " src-dir : directory where backup resides\n";
print " dest-dir: directory where backup should be restored to\n";
print " mode : " . join(', ', @RESTORE_MODES) . "\n";
print " version : \n";
print " - when run with 'list' parameter, displays available versions of incremental backups\n";
print " - if version is specified, tries to restore backup for specified version\n";
print " dry-run : check everything and exit without any changes\n\n";
exit(1);
}
mysql-mmm-2.2.1/sbin/mmm_mond 0000755 0000000 0000000 00000005503 11370745632 014642 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use threads;
use threads::shared;
use Config;
use English qw( PROGRAM_NAME );
use File::Basename;
use POSIX ':sys_wait_h';
use Proc::Daemon;
use Log::Log4perl qw(:easy);
Log::Log4perl->easy_init($INFO);
# Define version and protocol version
use constant MMM_VERSION => '2.2.1';
use constant MMM_PROTOCOL_VERSION => 1;
# Check perl for threads support
$Config{useithreads} or die "Recompile Perl with threads to run this program.";
# Include parts of the system
use MMM::Common::Angel;
use MMM::Common::Config;
use MMM::Common::Log;
use MMM::Common::PidFile;
use MMM::Monitor::Monitor;
use MMM::Monitor::NetworkChecker;
# Maybe we were just asked for our version
if (scalar(@ARGV) && $ARGV[0] eq "--version") {
printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION;
exit(0);
}
chdir('/');
umask(0022);
our $cluster_name = '';
my $postfix = '';
if (scalar(@ARGV) && $ARGV[0] =~ /^@(.*)/) {
shift(@ARGV);
$cluster_name = $1;
$postfix = "_$cluster_name";
$PROGRAM_NAME = basename($PROGRAM_NAME) . '-' . $cluster_name;
}
else {
$PROGRAM_NAME = basename($PROGRAM_NAME);
}
MMM::Common::Log::init("mmm_mon_log$postfix.conf", "mmm_mond$postfix");
# Read configuration
our $config = new MMM::Common::Config::;
$config->read("mmm_mon$postfix");
$config->check('MONITOR');
my $debug = $config->{debug};
MMM::Common::Log::debug() if ($debug);
INFO 'STARTING...';
our $monitor = new MMM::Monitor::Monitor::();
my $pidfilename = $config->{monitor}->{pid_path};
my $pidfile = new MMM::Common::PidFile:: $pidfilename;
# Check pid file
LOGDIE "Can't run second copy of ", $PROGRAM_NAME if ($pidfile->is_running());
WARN "Unclean start - found stale pid file!" if ($pidfile->exists());
unless ($debug) {
# Go to background
Proc::Daemon::Init();
# Set umask again
umask(0022);
# Init logging again to re-open fds
MMM::Common::Log::init("mmm_mon_log$postfix.conf", "mmm_mond$postfix");
}
# Init angel magic, which will restart us if we die unexpected
MMM::Common::Angel::Init($pidfile);
our $shutdown :shared = 0; # Shutdown flag
our $have_net :shared = 1; # Network status flag
# Set signal handlers
$SIG{INT} = \&ShutdownHandler;
$SIG{TERM} = \&ShutdownHandler;
$SIG{QUIT} = \&ShutdownHandler;
$SIG{PIPE} = 'IGNORE';
$SIG{CHLD} = \&ChildHandler;
if ($monitor->init()) {
$monitor->main();
}
INFO 'END';
exit(0);
#-----------------------------------------------------------------
sub ShutdownHandler() {
INFO "Signal received: exiting...";
$shutdown = 1;
}
#-----------------------------------------------------------------
sub ChildHandler {
local $!; # don't let waitpid() overwrite current error
while ((my $pid = waitpid(-1, WNOHANG)) > 0 && WIFEXITED($?)) {
DEBUG "Core: reaped child $pid" . ($? ? " with exit $?" : '');
}
$SIG{CHLD} = \&ChildHandler; # loathe sysV
}
mysql-mmm-2.2.1/sbin/mmm_backup 0000755 0000000 0000000 00000005004 11370745632 015146 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use English qw( PROGRAM_NAME );
use File::Basename;
use Log::Log4perl qw(:easy);
use Getopt::Long;
Log::Log4perl->easy_init( { level => $INFO, layout => '%p: %m%n' });
# Define version
use constant MMM_VERSION => '2.2.1';
# Include parts of the system
use MMM::Common::Config;
use MMM::Tools::Tools;
# Maybe we were just asked for our version
if (scalar(@ARGV) == 1 && $ARGV[0] eq "--version") {
printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION;
exit(0);
}
my $config_file = '';
my $host_name = '';
my $copy_method = '';
my $dest_dir = '';
print_usage() unless (
GetOptions(
'config=s' => \$config_file,
'host=s' => \$host_name,
'copy-method=s' => \$copy_method,
'dest-dir=s' => \$dest_dir
)
);
$config_file = 'mmm_tools' if ($config_file eq '');
# Read configuration
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('TOOLS');
print_usage("Invalid host name '$host_name'") unless (defined($config->{host}->{$host_name}));
my $host = $config->{host}->{$host_name};
$copy_method = $config->{default_copy_method} if ($copy_method eq '');
$dest_dir = $host->{backup_dir} if ($dest_dir eq '');
print_usage("Invalid copy method '$copy_method'") unless (defined($config->{copy_method}->{$copy_method}));
my $should_be_empty = (!$config->{copy_method}->{$copy_method}->{incremental});
print_usage("Invalid backup directory '$dest_dir'") unless (MMM::Tools::Tools::check_backup_destination($dest_dir, $should_be_empty));
die unless (MMM::Tools::Tools::check_ssh_connection($host_name));
die unless (MMM::Tools::Tools::create_remote_snapshot($host_name));
die unless (MMM::Tools::Tools::copy_clone_dirs($host_name, $copy_method, $dest_dir));
die unless (MMM::Tools::Tools::copy_from_remote($host_name, 'scp', $dest_dir, '_mmm'));
die unless (MMM::Tools::Tools::save_copy_method($dest_dir, $copy_method));
die unless (MMM::Tools::Tools::remove_remote_snapshot($host_name));
exit 0;
sub print_usage {
my $msg = shift;
print "$msg\n\n" if ($msg);
print "Usage: $0 [--config ] --host [--copy-method ] [--dest-dir ]\n";
if ($main::config) {
print "Where:\n";
printf(" host : %s\n", join(' | ', sort(keys(%{$main::config->{host}}))));
printf(" copy-method: %s (default: %s)\n", join(' | ', sort(keys(%{$main::config->{copy_method}}))), $config->{default_copy_method});
print " dest-dir : directory where data should be backed up to\n\n";
}
exit(1);
}
mysql-mmm-2.2.1/sbin/mmm_clone 0000755 0000000 0000000 00000013704 11370745632 015007 0 ustar root root #!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use English qw( PROGRAM_NAME );
use File::Basename;
use Log::Log4perl qw(:easy);
use Getopt::Long;
Log::Log4perl->easy_init( { level => $INFO, layout => '%p: %m%n' });
# Define version
use constant MMM_VERSION => '2.2.1';
# Include parts of the system
use MMM::Common::Config;
use MMM::Tools::Tools;
use MMM::Tools::MySQL;
# Maybe we were just asked for our version
if (scalar(@ARGV) == 1 && $ARGV[0] eq "--version") {
printf "%s %s\n", basename($PROGRAM_NAME), MMM_VERSION;
exit(0);
}
our @CLONE_MODES = qw(
master-master
master-slave
slave-slave
);
my $config_file = '';
my $host_name = '';
my $clone_mode = '';
my $copy_method = '';
my $dest_dir = '';
my $dry_run = 0;
print_usage() unless (
GetOptions(
'config=s' => \$config_file,
'host=s' => \$host_name,
'clone-mode=s' => \$clone_mode,
'copy-method=s' => \$copy_method,
'dest-dir=s' => \$dest_dir,
'dry-run' => \$dry_run
)
);
$config_file = 'mmm_tools' if ($config_file eq '');
# Read configuration
our $config = new MMM::Common::Config::;
$config->read($config_file);
$config->check('TOOLS');
# Check params and set defaults
print_usage("Invalid host name '$host_name'") unless (defined($config->{host}->{$host_name}));
print_usage("We can't clone ourselves") unless ($host_name ne $config->{this});
my $dest_host_info = $config->{host}->{$config->{this}};
$copy_method = $config->{default_copy_method} if ($copy_method eq '');
$dest_dir = $dest_host_info->{restore_dir} if ($dest_dir eq '');
print_usage("Unknown clone mode '$clone_mode'") unless (grep(/^$clone_mode$/, @CLONE_MODES));
print_usage("Invalid copy method '$copy_method'") unless (defined($config->{copy_method}->{$copy_method}));
print_usage("Only copy methods which create an exact copy can be used for cloning")
unless ($config->{copy_method}->{$copy_method}->{true_copy});
print_usage("Invalid destination directory '$dest_dir'") unless (MMM::Tools::Tools::check_restore_destination($dest_dir));
# Determine replication peer
my $peer_host = $host_name;
if ($clone_mode eq 'slave-slave') {
my $master_ip = MMM::Tools::MySQL::get_master_host($host_name);
die unless ($master_ip);
$peer_host = MMM::Tools::Tools::get_host_by_ip($master_ip);
}
unless (defined($config->{host}->{$peer_host})) {
LOGDIE "Unknown peer host '$peer_host'";
}
my $peer_host_info = $config->{host}->{$peer_host};
# Print info
my $label;
my $value;
format CLONE_INFO =
@<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<: @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
$label, $value
.
$FORMAT_NAME = 'CLONE_INFO';
$label = 'Source host';
$value = $host_name;
write;
$label = 'Destination dir';
$value = $dest_dir;
write;
$label = 'Dirs to clone';
$value = join(', ', sort(@{$config->{clone_dirs}}));
write;
$label = 'Clone mode';
$value = $clone_mode;
write;
$label = 'Copy method';
$value = $copy_method;
write;
$label = 'Replication peer';
$value = $peer_host;
write;
$label = 'Setup master-master replication';
$value = ($clone_mode eq 'master-master'? 'yes' : 'no');
write;
$label = 'Dry run';
$value = $dry_run ? 'yes' : 'no';
write;
# Stop here if this is a dry-run
exit 0 if ($dry_run);
# Stop mysql
die unless (MMM::Tools::MySQL::stop());
# Fetch snapshot from remote host
die unless (MMM::Tools::Tools::check_ssh_connection($host_name));
die unless (MMM::Tools::Tools::create_remote_snapshot($host_name));
die unless (MMM::Tools::Tools::copy_clone_dirs($host_name, $copy_method, $dest_dir));
die unless (MMM::Tools::Tools::copy_from_remote($host_name, 'scp', $dest_dir, '_mmm'));
die unless (MMM::Tools::Tools::save_copy_method($dest_dir, $copy_method));
die unless (MMM::Tools::Tools::remove_remote_snapshot($host_name));
# Load information from status file
my $status = MMM::Tools::Tools::load_status($dest_dir);
die unless (defined($status));
# Cleanup
die unless (MMM::Tools::Tools::cleanup($status, $dest_dir, $config->{clone_dirs}));
# Start MySQL server
die unless (MMM::Tools::MySQL::start());
# Setup replication
my %dest_replication_info = (
host => $config->{this},
master_host => $peer_host_info->{ip},
master_port => $peer_host_info->{mysql_port},
master_user => $peer_host_info->{replication_user},
master_pass => $peer_host_info->{replication_password},
);
if ($clone_mode eq 'master-slave' || $clone_mode eq 'master-master') {
$dest_replication_info{master_log} = $status->{master}->{'File'};
$dest_replication_info{master_pos} = $status->{master}->{'Position'};
die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info);
}
elsif ($clone_mode eq 'slave-slave') {
$dest_replication_info{master_log} = $status->{slave}->{'Relay_Master_Log_File'};
$dest_replication_info{master_pos} = $status->{slave}->{'Exec_Master_Log_Pos'};
die unless MMM::Tools::MySQL::change_master_to(\%dest_replication_info);
}
# Setup peer replication
if ($clone_mode eq 'master-master') {
my %peer_replication_info = (
host => $peer_host,
master_host => $dest_host_info->{ip},
master_port => $dest_host_info->{mysql_port},
master_user => $dest_host_info->{replication_user},
master_pass => $dest_host_info->{replication_password},
);
die unless MMM::Tools::MySQL::change_master_to(\%peer_replication_info);
}
INFO 'Clone operation finished!';
exit 0;
sub print_usage {
my $msg = shift;
print "$msg\n\n" if ($msg);
print "Usage: $0 [--config ] --host --clone-mode [--copy-method ] [--dest-dir ]\n";
if ($main::config) {
my @allowed_methods = grep { $main::config->{copy_method}->{$_}->{true_copy}} keys(%{$main::config->{copy_method}});
print "Where:\n";
printf(" host : %s\n", join(' | ', sort(keys(%{$main::config->{host}}))));
printf(" clone-mode : %s\n", join(' | ', sort(@CLONE_MODES)));
printf(" copy-method: %s (default: %s)\n", join(' | ', sort(@allowed_methods)), $config->{default_copy_method});
print " dest-dir : directory where data should be cloned to\n\n";
}
exit(1);
}