pax_global_header00006660000000000000000000000064132550257730014523gustar00rootroot0000000000000052 comment=abe11f9abb74bc5886012ff5cc9e209fb9b093c1 mha4mysql-manager-0.58/000077500000000000000000000000001325502577300150065ustar00rootroot00000000000000mha4mysql-manager-0.58/.gitignore000066400000000000000000000001721325502577300167760ustar00rootroot00000000000000Makefile inc/ .*.sw[pon] *.bak *.old *.log *.orig *.gz *.rpm Build _build/ META.yml MYMETA.yml blib/ pm_to_blib mha_*.cnf mha4mysql-manager-0.58/AUTHORS000066400000000000000000000001161325502577300160540ustar00rootroot00000000000000Developer and Maintainer: Yoshinori Matsunobu mha4mysql-manager-0.58/COPYING000066400000000000000000000431031325502577300160420ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. mha4mysql-manager-0.58/MANIFEST000066400000000000000000000114131325502577300161370ustar00rootroot00000000000000AUTHORS bin/masterha_check_repl bin/masterha_check_ssh bin/masterha_check_status bin/masterha_conf_host bin/masterha_manager bin/masterha_master_monitor bin/masterha_master_switch bin/masterha_secondary_check bin/masterha_stop COPYING debian/changelog debian/compat debian/control debian/copyright debian/docs debian/rules inc/Module/AutoInstall.pm inc/Module/Install.pm inc/Module/Install/AutoInstall.pm inc/Module/Install/Base.pm inc/Module/Install/Can.pm inc/Module/Install/Fetch.pm inc/Module/Install/Include.pm inc/Module/Install/Makefile.pm inc/Module/Install/Metadata.pm inc/Module/Install/Scripts.pm inc/Module/Install/Win32.pm inc/Module/Install/WriteAll.pm lib/MHA/Config.pm lib/MHA/DBHelper.pm lib/MHA/FileStatus.pm lib/MHA/HealthCheck.pm lib/MHA/ManagerAdmin.pm lib/MHA/ManagerAdminWrapper.pm lib/MHA/ManagerConst.pm lib/MHA/ManagerUtil.pm lib/MHA/MasterFailover.pm lib/MHA/MasterMonitor.pm lib/MHA/MasterRotate.pm lib/MHA/Server.pm lib/MHA/ServerManager.pm lib/MHA/SSHCheck.pm Makefile.PL MANIFEST This list of files META.yml README rpm/masterha_manager.spec samples/conf/app1.cnf samples/conf/masterha_default.cnf samples/scripts/master_ip_failover samples/scripts/master_ip_online_change samples/scripts/power_manager samples/scripts/send_report t/99-perlcritic.t t/perlcriticrc tests/intro.txt tests/run_suites.sh tests/t/bulk_tran_insert.pl tests/t/change_relay_log_info.sh tests/t/check tests/t/env.sh tests/t/force_start_m.sh tests/t/grant.sql tests/t/grant_nopass.sql tests/t/init.sh tests/t/insert.pl tests/t/insert_binary.pl tests/t/kill_m.sh tests/t/master_ip_failover tests/t/master_ip_failover_blank tests/t/mha_test.cnf.tmpl tests/t/mha_test_connect.cnf.tmpl tests/t/mha_test_err1.cnf.tmpl tests/t/mha_test_err2.cnf.tmpl tests/t/mha_test_ignore.cnf.tmpl tests/t/mha_test_latest.cnf.tmpl tests/t/mha_test_mm.cnf.tmpl tests/t/mha_test_mm_online.cnf.tmpl tests/t/mha_test_multi.cnf.tmpl tests/t/mha_test_multi_online.cnf.tmpl tests/t/mha_test_nopass.cnf.tmpl tests/t/mha_test_online.cnf.tmpl tests/t/mha_test_online_pass.cnf.tmpl tests/t/mha_test_pass.cnf.tmpl tests/t/mha_test_reset.cnf.tmpl tests/t/mha_test_ssh.cnf.tmpl tests/t/my-row.cnf tests/t/my.cnf tests/t/run.sh tests/t/run_bg.sh tests/t/run_tests tests/t/start_m.sh tests/t/start_s1.sh tests/t/start_s2.sh tests/t/start_s4.sh tests/t/stop_m.sh tests/t/stop_s1.sh tests/t/stop_s2.sh tests/t/stop_s4.sh tests/t/t_4tier.sh tests/t/t_4tier_subm_dead.sh tests/t/t_advisory_connect.sh tests/t/t_advisory_select.sh tests/t/t_apply_many_logs.sh tests/t/t_apply_many_logs2.sh tests/t/t_apply_many_logs3.sh tests/t/t_binary.sh tests/t/t_conf.sh tests/t/t_data_io_error.sh tests/t/t_dual_master_error.sh tests/t/t_filter_incorrect.sh tests/t/t_ignore_nostart.sh tests/t/t_ignore_recovery1.sh tests/t/t_ignore_recovery2.sh tests/t/t_ignore_recovery3.sh tests/t/t_ignore_recovery4.sh tests/t/t_ignore_start.sh tests/t/t_keep_relay_log_purge.sh tests/t/t_large_data.sh tests/t/t_large_data_bulk.sh tests/t/t_large_data_bulk_slow.sh tests/t/t_large_data_slow.sh tests/t/t_large_data_slow2.sh tests/t/t_large_data_sql_fail.sh tests/t/t_large_data_sql_stop.sh tests/t/t_large_data_tran.sh tests/t/t_latest_recovery1.sh tests/t/t_latest_recovery2.sh tests/t/t_latest_recovery3.sh tests/t/t_manual.sh tests/t/t_mm.sh tests/t/t_mm_3tier.sh tests/t/t_mm_3tier_subm_dead.sh tests/t/t_mm_normal.sh tests/t/t_mm_normal_skip_reset.sh tests/t/t_mm_noslaves.sh tests/t/t_mm_ro_fail.sh tests/t/t_mm_subm_dead.sh tests/t/t_mm_subm_dead_many.sh tests/t/t_mm_subm_dead_noslave.sh tests/t/t_needsync_1.sh tests/t/t_needsync_1_nocm.sh tests/t/t_needsync_1_nopass.sh tests/t/t_needsync_1_pass.sh tests/t/t_needsync_1_ssh.sh tests/t/t_needsync_2.sh tests/t/t_needsync_2_pass.sh tests/t/t_needsync_2_ssh.sh tests/t/t_needsync_fail.sh tests/t/t_needsync_flush.sh tests/t/t_needsync_flush2.sh tests/t/t_needsync_flush3.sh tests/t/t_needsync_flush_slave.sh tests/t/t_new_master_heavy.sh tests/t/t_new_master_heavy_wait.sh tests/t/t_no_relay_log.sh tests/t/t_normal_crash.sh tests/t/t_normal_crash_nocm.sh tests/t/t_online_3tier.sh tests/t/t_online_3tier_slave.sh tests/t/t_online_3tier_slave_keep.sh tests/t/t_online_busy.sh tests/t/t_online_filter.sh tests/t/t_online_mm.sh tests/t/t_online_mm_3tier.sh tests/t/t_online_mm_3tier_slave.sh tests/t/t_online_mm_skip_reset.sh tests/t/t_online_normal.sh tests/t/t_online_slave.sh tests/t/t_online_slave_fail.sh tests/t/t_online_slave_pass.sh tests/t/t_online_slave_sql_stop.sh tests/t/t_recover_master_fail.sh tests/t/t_recover_slave_fail.sh tests/t/t_recover_slave_fail2.sh tests/t/t_recover_slave_ok.sh tests/t/t_save_master_log.sh tests/t/t_save_master_log_pass.sh tests/t/t_save_master_log_ssh.sh tests/t/t_slave_incorrect.sh tests/t/t_slave_sql_start.sh tests/t/t_slave_sql_start2.sh tests/t/t_slave_sql_start3.sh tests/t/t_slave_stop.sh tests/t/tran_insert.pl tests/t/waitpid mha4mysql-manager-0.58/MANIFEST.SKIP000066400000000000000000000003511325502577300167030ustar00rootroot00000000000000\bRCS\b \bCVS\b ^MANIFEST\. ^MYMETA\.json ^MYMETA\.yml ^Makefile$ ~$ \.log$ \.old$ \.bak$ \.orig$ ^blib/ ^pm_to_blib ^MakeMaker-\d \.gz$ \.rpm$ \.cvsignore \.swp \.shipit ^t/9\d_.*\.t \.svn/ \.git/ \.gitignore$ ^author/ mha_.*\.cnf$ mha4mysql-manager-0.58/Makefile.PL000077500000000000000000000015161325502577300167660ustar00rootroot00000000000000use inc::Module::Install; name 'mha4mysql-manager'; version_from 'lib/MHA/ManagerConst.pm'; requires 'DBI'; requires 'DBD::mysql'; requires 'Time::HiRes'; requires 'Config::Tiny'; requires 'Log::Dispatch'; requires 'Parallel::ForkManager'; requires 'MHA::NodeConst'; license 'GPL v2'; author 'Yoshinori Matsunobu '; if ($ENV{REWRITE_SHEBANG}) { mkdir 'xbin'; for my $bin (glob 'bin/*') { (my $xbin = $bin) =~ s{^bin/}{xbin/}; open my $in, '<', $bin or die $!; open my $out, '>', $xbin or die $!; while (<$in>) { s|^#!/usr/bin/env perl|#!perl|; # so MakeMaker can fix it print $out $_; } close $in; close $out; } install_script(glob 'xbin/*'); } else { install_script(glob 'bin/*'); } auto_install; WriteAll; mha4mysql-manager-0.58/README000066400000000000000000000003641325502577300156710ustar00rootroot00000000000000mha4mysql-manager - Master High Availability Manager and tools for MySQL (MHA) for automating master failover and fast master switch. This package contains manager scripts. See https://github.com/yoshinorim/mha4mysql-manager/wiki for details. mha4mysql-manager-0.58/bin/000077500000000000000000000000001325502577300155565ustar00rootroot00000000000000mha4mysql-manager-0.58/bin/masterha_check_repl000077500000000000000000000037131325502577300214730ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use Pod::Usage; use Getopt::Long qw(:config pass_through); use MHA::MasterMonitor; use MHA::ManagerConst; my $help; my $version; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, ); if ($version) { print "masterha_check_repl version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } my $exit_code = 1; $exit_code = MHA::MasterMonitor::main( "--interactive=0", "--check_only", "--check_repl_health", @ARGV ); if ( $exit_code == 0 ) { print "\nMySQL Replication Health is OK.\n"; } else { print "\nMySQL Replication Health is NOT OK!\n"; } exit $exit_code; # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_check_repl - Checking MySQL replication health =head1 SYNOPSIS masterha_check_repl --conf=/usr/local/masterha/conf/app1.cnf See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_check_repl) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_check_repl) for details. mha4mysql-manager-0.58/bin/masterha_check_ssh000077500000000000000000000033631325502577300213270ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use Getopt::Long qw(:config pass_through); use Pod::Usage; use MHA::SSHCheck; my $help; my $version; GetOptions( 'help' => \$help, 'version' => \$version, ); if ($version) { print "masterha_check_ssh version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } exit MHA::SSHCheck::main(@ARGV); # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_check_ssh - Checking SSH connections =head1 SYNOPSIS masterha_check_ssh --global_conf=/etc/masterha_default.cnf --conf=/etc/conf/masterha/app1.cnf See online reference (http://code.google.com/p/mysql-master-ha/wiki/Requirements#SSH_public_key_authentication) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/Requirements#SSH_public_key_authentication) for details. mha4mysql-manager-0.58/bin/masterha_check_status000077500000000000000000000035111325502577300220500ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use Getopt::Long qw(:config pass_through); use Pod::Usage; use MHA::ManagerConst; use MHA::ManagerAdminWrapper; my $help; my $version; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, ); if ($version) { print "masterha_check_status version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } my $rc = 0; eval { $rc = MHA::ManagerAdminWrapper::check_status(@ARGV); }; if ($@) { $rc = 1; } exit $rc; # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_check_status - Returns target MySQL master status monitored by MHA Manager =head1 SYNOPSIS masterha_check_status --conf=/usr/local/masterha/conf/app1.cnf See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_check_status) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_check_status) for details. mha4mysql-manager-0.58/bin/masterha_conf_host000077500000000000000000000062011325502577300213510ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use Getopt::Long; use Pod::Usage; use MHA::Config; my $help; my $version; my $command; my $conf; my $block; my $hostname; my $params; GetOptions( 'help' => \$help, 'version' => \$version, 'command=s' => \$command, 'conf=s' => \$conf, 'block=s' => \$block, 'hostname=s' => \$hostname, 'params=s' => \$params, ); if ($version) { print "masterha_conf_host version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } if ( !$command ) { die "--command= must be set.\n"; } if ( !$conf ) { die "--conf= must be set.\n"; } if ( !-f $conf ) { die "$conf does not exist!\n"; } if ($block) { if ( $block !~ m/^server/ ) { $block = "server" . $block; } } else { $block = "server_" . $hostname if ($hostname); } unless ($block) { die "--block= must be set.\n"; } if ( $command eq "add" ) { unless ($hostname) { die "--hostname= must be set.\n"; } my @param_array; if ($params) { @param_array = split( /;/, $params ); } MHA::Config::add_block_and_save( $conf, $block, $hostname, \@param_array ); } elsif ( $command eq "delete" || $command eq "remove" ) { MHA::Config::delete_block_and_save( $conf, $block ); } else { pod2usage(1); } # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_conf_host - Adding new host entry to, or removing existing host entry from a config file =head1 SYNOPSIS masterha_conf_host --command=add --conf=/etc/conf/masterha/app1.cnf --hostname=db101 The following lines will be added to the conf file. [server_db101] hostname=db101 masterha_conf_host --command=add --conf=/etc/conf/masterha/app1.cnf --hostname=db101 --block=100 --params="no_master=1;ignore_fail=1" The following lines will be added to the conf file. [server_100] hostname=db101 no_master=1 ignore_fail=1 masterha_conf_host --command=delete --conf=/etc/conf/masterha/app1.cnf --block=server100 Then entire block [server100] will be removed. See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_conf_host) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_conf_host) for details. mha4mysql-manager-0.58/bin/masterha_manager000077500000000000000000000047251325502577300210120ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Getopt::Long qw(:config pass_through); use Pod::Usage; use MHA::MasterMonitor; use MHA::MasterFailover; use MHA::ManagerConst; my $help; my $version; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, ); if ($version) { print "masterha_manager version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } my @ORIG_ARGV = @ARGV; my ( $exit_code, $dead_master, $ssh_reachable ) = MHA::MasterMonitor::main( "--interactive=0", @ARGV ); if ( $exit_code && $exit_code != $MHA::ManagerConst::MASTER_DEAD_RC ) { exit $exit_code; } if ( !$dead_master->{hostname} || !$dead_master->{ip} || !$dead_master->{port} || !defined($ssh_reachable) ) { exit 1; } @ARGV = @ORIG_ARGV; $exit_code = MHA::MasterFailover::main( "--master_state=dead", "--interactive=0", "--dead_master_host=$dead_master->{hostname}", "--dead_master_ip=$dead_master->{ip}", "--dead_master_port=$dead_master->{port}", "--ssh_reachable=$ssh_reachable", @ARGV ); exit $exit_code; # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_manager - Monitoring MySQL master server availability and do failover if it detects master failure =head1 SYNOPSIS masterha_manager --global_conf=/etc/masterha_default.cnf --conf=/usr/local/masterha/conf/app1.cnf See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_manager) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_manager) for details. mha4mysql-manager-0.58/bin/masterha_master_monitor000077500000000000000000000041651325502577300224400ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Getopt::Long qw(:config pass_through); use Pod::Usage; use MHA::MasterMonitor; use MHA::ManagerConst; my $help; my $version; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, ); if ($version) { print "masterha_master_monitor version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } my ( $exit_code, $dead_master, $ssh_reachable ) = MHA::MasterMonitor::main( "--monitor_only", @ARGV ); if ( $dead_master->{hostname} ) { print "Master $dead_master->{hostname} is dead!\n"; print "IP Address: $dead_master->{ip} "; print "Port: $dead_master->{port}\n"; } if ( $exit_code && $exit_code eq "0" ) { if ($ssh_reachable) { print "SSH: reachable\n"; } else { print "SSH: NOT reachable\n"; } exit 0; } exit 1 if ( !defined($exit_code) ); exit $exit_code; # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_master_monitor - Monitoring MySQL master server availability =head1 SYNOPSIS masterha_master_monitor --global_conf=/etc/masterha_default.cnf --conf=/usr/local/masterha/app1.cnf =head1 DESCRIPTION masterha_master_monitor is intended to be invoked from masterha_manager. mha4mysql-manager-0.58/bin/masterha_master_switch000077500000000000000000000045051325502577300222500ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use Getopt::Long qw(:config pass_through); use Pod::Usage; use MHA::MasterMonitor; use MHA::MasterFailover; use MHA::MasterRotate; use MHA::ManagerConst; my $master_state = ""; my $help; my $version; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, 'master_state=s' => \$master_state ); my $exit_code = 1; if ($version) { print "masterha_master_switch version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } if ( $master_state eq "dead" ) { $exit_code = MHA::MasterFailover::main(@ARGV); } elsif ( $master_state eq "alive" ) { $exit_code = MHA::MasterRotate::main(@ARGV); } else { pod2usage(1); } exit $exit_code; # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_master_switch - Switching MySQL master server to one of other slave servers =head1 SYNOPSIS # For master failover masterha_master_switch --master_state=dead --global_conf=/etc/masterha_default.cnf --conf=/usr/local/masterha/conf/app1.cnf --dead_master_host=host1 # For online master switch masterha_master_switch --master_state=alive --global_conf=/etc/masterha_default.cnf --conf=/usr/local/masterha/conf/app1.cnf See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_master_switch) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_master_switch) for details. mha4mysql-manager-0.58/bin/masterha_secondary_check000077500000000000000000000120641325502577300225170ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Getopt::Long; use Pod::Usage; use MHA::ManagerConst; my @monitoring_servers; my ( $help, $version, $ssh_user, $ssh_port, $ssh_options, $master_host, $master_ip, $master_port, $master_user, $master_password, $ping_type ); my $timeout = 5; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, 'secondary_host=s' => \@monitoring_servers, 'user=s' => \$ssh_user, 'port=s' => \$ssh_port, 'options=s' => \$ssh_options, 'master_host=s' => \$master_host, 'master_ip=s' => \$master_ip, 'master_port=i' => \$master_port, 'master_user=s' => \$master_user, 'master_password=s' => \$master_password, 'ping_type=s' => \$ping_type, 'timeout=i' => \$timeout, ); if ($version) { print "masterha_secondary_check version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } unless ($master_host) { pod2usage(1); } sub exit_by_signal { exit 1; } local $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = \&exit_by_signal; $ssh_user = "root" unless ($ssh_user); $ssh_port = 22 unless ($ssh_port); $master_port = 3306 unless ($master_port); if ($ssh_options) { $MHA::ManagerConst::SSH_OPT_CHECK = $ssh_options; } $MHA::ManagerConst::SSH_OPT_CHECK =~ s/VAR_CONNECT_TIMEOUT/$timeout/; # 0: master is not reachable from all monotoring servers # 1: unknown errors # 2: at least one of monitoring servers is not reachable from this script # 3: master is reachable from at least one of monitoring servers my $exit_code = 0; foreach my $monitoring_server (@monitoring_servers) { my $ssh_user_host = $ssh_user . '@' . $monitoring_server; my $command = "ssh $MHA::ManagerConst::SSH_OPT_CHECK -p $ssh_port $ssh_user_host \"perl -e " . "\\\"use IO::Socket::INET; my \\\\\\\$sock = IO::Socket::INET->new" . "(PeerAddr => \\\\\\\"$master_host\\\\\\\", PeerPort=> $master_port, " . "Proto =>'tcp', Timeout => $timeout); if(\\\\\\\$sock) { close(\\\\\\\$sock); " . "exit 3; } exit 0;\\\" \""; my $ret = system($command); $ret = $ret >> 8; if ( $ret == 0 ) { print "Monitoring server $monitoring_server is reachable, Master is not reachable from $monitoring_server. OK.\n"; next; } if ( $ret == 3 ) { if ( defined $ping_type && $ping_type eq $MHA::ManagerConst::PING_TYPE_INSERT ) { my $ret_insert; my $command_insert = "ssh $MHA::ManagerConst::SSH_OPT_CHECK -p $ssh_port $ssh_user_host \'" . "/usr/bin/mysql -u$master_user -p$master_password -h$master_host -P$master_port " . "-e \"CREATE DATABASE IF NOT EXISTS infra; " . "CREATE TABLE IF NOT EXISTS infra.chk_masterha (\\`key\\` tinyint NOT NULL primary key,\\`val\\` int(10) unsigned NOT NULL DEFAULT '0'\); " . "INSERT INTO infra.chk_masterha values (1,unix_timestamp()) ON DUPLICATE KEY UPDATE val=unix_timestamp()\"\'"; my $sigalrm_timeout = 3; eval { local $SIG{ALRM} = sub { die "timeout.\n"; }; alarm $sigalrm_timeout; $ret_insert = system($command_insert); $ret_insert = $ret_insert >> 8; alarm 0; }; if ( $@ || $ret_insert != 0 ) { print "Monitoring server $monitoring_server is reachable, Master is not writable from $monitoring_server. OK.\n"; next; } } print "Master is reachable from $monitoring_server!\n"; $exit_code = 3; last; } else { print "Monitoring server $monitoring_server is NOT reachable!\n"; $exit_code = 2; last; } } exit $exit_code; # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_secondary_check - Checking master availability from additional network routes =head1 SYNOPSIS masterha_secondary_check -s secondary_host1 -s secondary_host2 .. --user=ssh_username --master_host=host --master_ip=ip --master_port=port See online reference (http://code.google.com/p/mysql-master-ha/wiki/Parameters#secondary_check_script) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/Parameters#secondary_check_script) for details. mha4mysql-manager-0.58/bin/masterha_stop000077500000000000000000000033131325502577300203550ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA use strict; use warnings FATAL => 'all'; use Getopt::Long qw(:config pass_through); use Pod::Usage; use MHA::ManagerConst; use MHA::ManagerAdminWrapper; my $help; my $version; $| = 1; GetOptions( 'help' => \$help, 'version' => \$version, ); if ($version) { print "masterha_stop version $MHA::ManagerConst::VERSION.\n"; exit 0; } if ($help) { pod2usage(0); } exit MHA::ManagerAdminWrapper::stop_manager(@ARGV); # ############################################################################ # Documentation # ############################################################################ =pod =head1 NAME masterha_stop - Stopping MHA Manager process =head1 SYNOPSIS masterha_stop --conf=/usr/local/masterha/conf/app1.cnf See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_stop) for details. =head1 DESCRIPTION See online reference (http://code.google.com/p/mysql-master-ha/wiki/masterha_stop) for details. mha4mysql-manager-0.58/debian/000077500000000000000000000000001325502577300162305ustar00rootroot00000000000000mha4mysql-manager-0.58/debian/changelog000066400000000000000000000027201325502577300201030ustar00rootroot00000000000000mha4mysql-manager (0.58-0) squeeze; urgency=low * Several bugfixes * IPV6 support * super_read_only support * enforce_storage_engine support * Version 0.58 * With the help of Marcelo Altmann -- Kenny Gryp Thu, 22 Mar 2018 00:00:00 +0000 mha4mysql-manager (0.56-0) squeeze; urgency=low * (Note: All changelogs are written here: http://code.google.com/p/mysql-master-ha/wiki/ReleaseNotes ) * Version 0.56 -- Yoshinori Matsunobu Tue, 1 Apr 2014 00:00:00 +0000 mha4mysql-manager (0.55-0) squeeze; urgency=low * Version 0.55 -- Yoshinori Matsunobu Wed, 12 Dec 2012 00:00:00 +0000 mha4mysql-manager (0.54-0) squeeze; urgency=low * Version 0.54 -- Yoshinori Matsunobu Sat, 1 Dec 2012 00:00:00 +0000 mha4mysql-manager (0.53) stable; urgency=low * Version 0.53 -- Yoshinori Matsunobu Mon, 9 Jan 2012 00:00:00 +0000 mha4mysql-manager (0.52) stable; urgency=low * Version 0.52 -- Yoshinori Matsunobu Fri, 16 Sep 2011 00:00:00 +0000 mha4mysql-manager (0.51) stable; urgency=low * Version 0.51 -- Yoshinori Matsunobu Thu, 18 Aug 2011 00:00:00 +0000 mha4mysql-manager (0.50) stable; urgency=low * Version 0.50 -- Yoshinori Matsunobu Sat, 23 Jul 2011 00:00:00 +0000 mha4mysql-manager-0.58/debian/compat000066400000000000000000000000021325502577300174260ustar00rootroot000000000000007 mha4mysql-manager-0.58/debian/control000066400000000000000000000011011325502577300176240ustar00rootroot00000000000000Source: mha4mysql-manager Section: perl Priority: optional Build-Depends: debhelper (>= 7.2.13) Build-Depends-Indep: perl Maintainer: Yoshinori Matsunobu Standards-Version: 3.9.1 Homepage: http://code.google.com/p/mysql-master-ha/ Package: mha4mysql-manager Architecture: all Depends: ${misc:Depends}, ${perl:Depends}, libdbi-perl, libdbd-mysql-perl, libconfig-tiny-perl, liblog-dispatch-perl, libparallel-forkmanager-perl, mha4mysql-node (>= 0.54) Description: Master High Availability Manager and Tools for MySQL, Manager Package mha4mysql-manager-0.58/debian/copyright000066400000000000000000000006051325502577300201640ustar00rootroot00000000000000It was downloaded from http://code.google.com/p/mysql-master-ha/ Upstream Author(s): Yoshinori Matsunobu Copyright: Copyright (C) 2011 DeNA Co.,Ltd. License: GPLv2 See `cat /usr/share/common-licenses/GPL-2`. The Debian packaging is done by Yoshinori Matsunobu in 2011, and is licensed under the GPL v2 license. mha4mysql-manager-0.58/debian/docs000066400000000000000000000000171325502577300171010ustar00rootroot00000000000000README AUTHORS mha4mysql-manager-0.58/debian/rules000077500000000000000000000000361325502577300173070ustar00rootroot00000000000000#!/usr/bin/make -f %: dh $@ mha4mysql-manager-0.58/lib/000077500000000000000000000000001325502577300155545ustar00rootroot00000000000000mha4mysql-manager-0.58/lib/MHA/000077500000000000000000000000001325502577300161615ustar00rootroot00000000000000mha4mysql-manager-0.58/lib/MHA/Config.pm000066400000000000000000000421621325502577300177310ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::Config; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use English qw(-no_match_vars); use Config::Tiny; use Log::Dispatch; use MHA::Server; use MHA::NodeUtil; use MHA::ManagerConst; my @PARAM_ARRAY = qw/ hostname ip port ssh_host ssh_ip ssh_port ssh_connection_timeout ssh_options node_label candidate_master no_master ignore_fail skip_init_ssh_check skip_reset_slave user password repl_user repl_password disable_log_bin master_pid_file handle_raw_binlog ssh_user remote_workdir master_binlog_dir log_level manager_workdir manager_log check_repl_delay check_repl_filter latest_priority multi_tier_slave ping_interval ping_type secondary_check_script master_ip_failover_script master_ip_online_change_script shutdown_script report_script init_conf_load_script client_bindir client_libdir use_gtid_auto_pos/; my %PARAM; for (@PARAM_ARRAY) { $PARAM{$_} = 1; } sub new { my $class = shift; my $self = { file => undef, globalfile => undef, logger => undef, @_, }; return bless $self, $class; } sub parse_server { my $self = shift; my $param_arg = shift; my $default = shift; my %value; foreach my $key ( sort keys(%$param_arg) ) { unless ( exists( $PARAM{$key} ) ) { croak "Parameter name $key is invalid!\n"; } } $value{hostname} = $param_arg->{hostname}; $value{ip} = $param_arg->{ip}; $value{ip} = MHA::NodeUtil::get_ip( $value{hostname} ) unless ( $value{ip} ); $value{node_label} = $param_arg->{node_label}; $value{port} = $param_arg->{port}; if ( !defined( $value{port} ) ) { $value{port} = $default->{port}; $value{port} = 3306 unless ( $value{port} ); } $value{ssh_host} = $param_arg->{ssh_host}; if ( !defined( $value{ssh_host} ) ) { $value{ssh_host} = $value{hostname}; } if ( $value{hostname} && $value{ssh_host} && $value{ssh_host} eq $value{hostname} ) { $value{ssh_ip} = $value{ip}; } else { $value{ssh_ip} = MHA::NodeUtil::get_ip( $value{ssh_host} ); } $value{ssh_port} = $param_arg->{ssh_port}; if ( !defined( $value{ssh_port} ) ) { $value{ssh_port} = $default->{ssh_port}; $value{ssh_port} = 22 unless ( $value{ssh_port} ); } $value{ssh_options} = $param_arg->{ssh_options}; if ( $value{ssh_options} ) { $value{ssh_options} =~ s/['"]//g; if ( $value{ssh_options} ) { $MHA::ManagerConst::USE_SSH_OPTIONS = 1; $MHA::ManagerConst::SSH_OPT_ALIVE = $MHA::ManagerConst::SSH_OPT_ALIVE_DEFAULT . " $value{ssh_options}"; $MHA::NodeConst::SSH_OPT_ALIVE = $MHA::ManagerConst::SSH_OPT_ALIVE_DEFAULT . " $value{ssh_options}"; $MHA::ManagerConst::SSH_OPT_CHECK = $MHA::ManagerConst::SSH_OPT_CHECK_DEFAULT . " $value{ssh_options}"; } } $value{ssh_connection_timeout} = $param_arg->{ssh_connection_timeout}; if ( !defined( $value{ssh_connection_timeout} ) ) { $value{ssh_connection_timeout} = $default->{ssh_connection_timeout}; $value{ssh_connection_timeout} = 5 unless ( $value{ssh_connection_timeout} ); } check_positive_int( "ssh_connection_timeout", $value{ssh_connection_timeout} ); $MHA::ManagerConst::SSH_OPT_CHECK =~ s/VAR_CONNECT_TIMEOUT/$value{ssh_connection_timeout}/; $value{user} = $param_arg->{user}; if ( !defined( $value{user} ) ) { $value{user} = $default->{user}; $value{user} = 'root' unless ( $value{user} ); } $value{password} = $param_arg->{password}; if ( !defined( $value{password} ) ) { $value{password} = $default->{password}; $value{password} = '' if ( !defined( $value{password} ) ); } $value{repl_user} = $param_arg->{repl_user}; if ( !defined( $value{repl_user} ) ) { $value{repl_user} = $default->{repl_user}; } $value{repl_password} = $param_arg->{repl_password}; if ( !defined( $value{repl_password} ) ) { $value{repl_password} = $default->{repl_password}; } $value{master_pid_file} = $param_arg->{master_pid_file}; if ( !defined( $value{master_pid_file} ) ) { $value{master_pid_file} = $default->{master_pid_file}; } $value{log_level} = $param_arg->{log_level}; if ( !defined( $value{log_level} ) ) { $value{log_level} = $default->{log_level}; $value{log_level} = 'info' unless ( $value{log_level} ); } $value{init_conf_load_script} = $param_arg->{init_conf_load_script}; if ( !defined( $value{init_conf_load_script} ) ) { $value{init_conf_load_script} = $default->{init_conf_load_script}; } $value{secondary_check_script} = $param_arg->{secondary_check_script}; if ( !defined( $value{secondary_check_script} ) ) { $value{secondary_check_script} = $default->{secondary_check_script}; } $value{shutdown_script} = $param_arg->{shutdown_script}; if ( !defined( $value{shutdown_script} ) ) { $value{shutdown_script} = $default->{shutdown_script}; } $value{report_script} = $param_arg->{report_script}; if ( !defined( $value{report_script} ) ) { $value{report_script} = $default->{report_script}; } $value{master_ip_failover_script} = $param_arg->{master_ip_failover_script}; if ( !defined( $value{master_ip_failover_script} ) ) { $value{master_ip_failover_script} = $default->{master_ip_failover_script}; } # For online master switch only $value{master_ip_online_change_script} = $param_arg->{master_ip_online_change_script}; if ( !defined( $value{master_ip_online_change_script} ) ) { $value{master_ip_online_change_script} = $default->{master_ip_online_change_script}; } $value{disable_log_bin} = $param_arg->{disable_log_bin}; if ( !defined( $value{disable_log_bin} ) ) { $value{disable_log_bin} = $default->{disable_log_bin}; $value{disable_log_bin} = 0 if ( !defined( $value{disable_log_bin} ) ); } $value{handle_raw_binlog} = $param_arg->{handle_raw_binlog}; if ( !defined( $value{handle_raw_binlog} ) ) { $value{handle_raw_binlog} = $default->{handle_raw_binlog}; $value{handle_raw_binlog} = 1 if ( !defined( $value{handle_raw_binlog} ) ); } $value{check_repl_delay} = $param_arg->{check_repl_delay}; if ( !defined( $value{check_repl_delay} ) ) { $value{check_repl_delay} = $default->{check_repl_delay}; $value{check_repl_delay} = 1 if ( !defined( $value{check_repl_delay} ) ); } $value{check_repl_filter} = $param_arg->{check_repl_filter}; if ( !defined( $value{check_repl_filter} ) ) { $value{check_repl_filter} = $default->{check_repl_filter}; $value{check_repl_filter} = 1 if ( !defined( $value{check_repl_filter} ) ); } $value{latest_priority} = $param_arg->{latest_priority}; if ( !defined( $value{latest_priority} ) ) { $value{latest_priority} = $default->{latest_priority}; $value{latest_priority} = 1 if ( !defined( $value{latest_priority} ) ); } $value{multi_tier_slave} = $param_arg->{multi_tier_slave}; if ( !defined( $value{multi_tier_slave} ) ) { $value{multi_tier_slave} = $default->{multi_tier_slave}; $value{multi_tier_slave} = 0 if ( !defined( $value{multi_tier_slave} ) ); } $value{skip_reset_slave} = $param_arg->{skip_reset_slave}; if ( !defined( $value{skip_reset_slave} ) ) { $value{skip_reset_slave} = $default->{skip_reset_slave}; $value{skip_reset_slave} = 0 if ( !defined( $value{skip_reset_slave} ) ); } $value{use_gtid_auto_pos} = $param_arg->{use_gtid_auto_pos}; if ( !defined( $value{use_gtid_auto_pos} ) ) { $value{use_gtid_auto_pos} = $default->{use_gtid_auto_pos}; $value{use_gtid_auto_pos} = 1 if ( !defined( $value{use_gtid_auto_pos} ) ); } $value{master_binlog_dir} = $param_arg->{master_binlog_dir}; if ( !defined( $value{master_binlog_dir} ) ) { $value{master_binlog_dir} = $default->{master_binlog_dir}; $value{master_binlog_dir} = "/var/lib/mysql,/var/log/mysql" unless ( $value{master_binlog_dir} ); } $value{master_binlog_dir} =~ s/\s//g if ( $value{master_binlog_dir} ); $value{manager_workdir} = $param_arg->{manager_workdir}; if ( !defined( $value{manager_workdir} ) ) { $value{manager_workdir} = $default->{manager_workdir}; $value{manager_workdir} = "/var/tmp" unless ( $value{manager_workdir} ); } $value{manager_log} = $param_arg->{manager_log}; if ( !defined( $value{manager_log} ) ) { $value{manager_log} = $default->{manager_log}; } $value{remote_workdir} = $param_arg->{remote_workdir}; if ( !defined( $value{remote_workdir} ) ) { $value{remote_workdir} = $default->{remote_workdir}; $value{remote_workdir} = "/var/tmp" unless ( $value{remote_workdir} ); } $value{ssh_user} = $param_arg->{ssh_user}; if ( !defined( $value{ssh_user} ) ) { $value{ssh_user} = $default->{ssh_user}; $value{ssh_user} = getpwuid($>) unless ( $value{ssh_user} ); } $value{candidate_master} = $param_arg->{candidate_master}; $value{candidate_master} = 0 if ( !defined( $value{candidate_master} ) ); $value{no_master} = $param_arg->{no_master}; $value{no_master} = 0 if ( !defined( $value{no_master} ) ); $value{ignore_fail} = $param_arg->{ignore_fail}; $value{ignore_fail} = 0 if ( !defined( $value{ignore_fail} ) ); $value{skip_init_ssh_check} = $param_arg->{skip_init_ssh_check}; $value{skip_init_ssh_check} = 0 if ( !defined( $value{skip_init_ssh_check} ) ); $value{ping_type} = $param_arg->{ping_type}; if ( !defined( $value{ping_type} ) ) { $value{ping_type} = $default->{ping_type}; $value{ping_type} = $MHA::ManagerConst::PING_TYPE_SELECT if ( !defined( $value{ping_type} ) ); } $value{ping_type} = uc( $value{ping_type} ); croak "Parameter ping_type must be either '$MHA::ManagerConst::PING_TYPE_CONNECT' or '$MHA::ManagerConst::PING_TYPE_SELECT' or '$MHA::ManagerConst::PING_TYPE_INSERT'. Current value: $value{ping_type}\n" if ( $value{ping_type} ne $MHA::ManagerConst::PING_TYPE_CONNECT && $value{ping_type} ne $MHA::ManagerConst::PING_TYPE_SELECT && $value{ping_type} ne $MHA::ManagerConst::PING_TYPE_INSERT ); $value{ping_interval} = $param_arg->{ping_interval}; if ( !defined( $value{ping_interval} ) ) { $value{ping_interval} = $default->{ping_interval}; $value{ping_interval} = 3 if ( !defined( $value{ping_interval} ) ); } check_positive_int( "ping_interval", $value{ping_interval} ); $value{client_bindir} = $param_arg->{client_bindir}; if ( !defined( $value{client_bindir} ) ) { $value{client_bindir} = $default->{client_bindir}; } $value{client_libdir} = $param_arg->{client_libdir}; if ( !defined( $value{client_libdir} ) ) { $value{client_libdir} = $default->{client_libdir}; } my $server = new MHA::Server(); foreach my $key ( keys(%PARAM) ) { if ( $value{$key} ) { $value{$key} =~ s/^['"]?(.*)['"]$/$1/; } $server->{$key} = $value{$key}; } # set escaped_user and escaped_password foreach my $key ( 'user', 'password' ) { my $new_key = "escaped_" . $key; my $new_mysql_key = "mysql_escaped_" . $key; if ( $server->{$key} ) { $server->{$new_key} = MHA::NodeUtil::escape_for_shell( $server->{$key} ); $server->{$new_mysql_key} = MHA::NodeUtil::escape_for_mysql_command( $server->{$key} ); } else { $server->{$new_key} = $server->{$new_mysql_key} = ""; } } return $server; } sub check_positive_int($$) { my $param_name = shift; my $param_value = shift; croak "Parameter $param_name must be positive integer! current value: $param_value\n" if ( $param_value !~ /\d/ || $param_value =~ /\D/ || $param_value < 1 ); } sub read_config($) { my $self = shift; my $log = $self->{logger}; my @servers = (); my @binlog_servers = (); my $global_configfile = $self->{globalfile}; my $configfile = $self->{file}; my $sd; if ( -f $global_configfile ) { my $global_cfg = Config::Tiny->read($global_configfile) or croak "Unable to parse/read configuration file: $global_configfile: $!\n"; $log->info("Reading default configuration from $self->{globalfile}..") if ($log); $sd = $self->parse_server_default( $global_cfg->{"server default"} ); } else { $log->warning( "Global configuration file $self->{globalfile} not found. Skipping.") if ($log); $sd = new MHA::Server(); } my $cfg = Config::Tiny->read($configfile) or croak "$configfile:$!\n"; $log->info("Reading application default configuration from $self->{file}..") if ($log); # Read application default settings $sd = $self->parse_server( $cfg->{"server default"}, $sd ); if ( defined( $sd->{init_conf_load_script} ) ) { $log->info( "Updating application default configuration from " . $sd->{init_conf_load_script} . ".." ) if ($log); my @rows = `$sd->{init_conf_load_script}`; my $param; foreach my $row (@rows) { chomp($row); my ( $name, $value ) = split( /=/, $row ); $param->{$name} = $value; } $sd = $self->parse_server( $param, $sd ); } $log->info("Reading server configuration from $self->{file}..") if ($log); my @blocks = sort keys(%$cfg); foreach my $block (@blocks) { next if ( $block eq "server default" ); if ( $block !~ /^server\S+/ && $block !~ /^binlog\S+/ ) { my $msg = "Block name \"$block\" is invalid. Block name must be \"server default\" or start from \"server\"(+ non-whitespace characters)."; $log->error($msg) if ($log); croak($msg); } my $server = $self->parse_server( $cfg->{$block}, $sd ); $server->{id} = $block; if ( $block =~ /^server\S+/ ) { push( @servers, $server ); } elsif ( $block =~ /^binlog\S+/ ) { push( @binlog_servers, $server ); } } my @tmp; foreach (@servers) { push @tmp, [ $1, $_ ] if ( $_->{id} =~ m/^server\D*([\d]+).*/ ); } # If all IDs are integers, sort by intergers if ( $#servers == $#tmp ) { @servers = map { $_->[1] } ( sort { $a->[0] <=> $b->[0] } @tmp ); } unless (@servers) { my $msg = "No server is defined in configurations file. Check configurations for details"; $log->error($msg) if ($log); croak($msg); } # check hostname exists for ( my $i = 0 ; $i <= $#servers ; $i++ ) { unless ( $servers[$i]->{hostname} ) { my $msg = sprintf( "Server %s does not have hostname! Check configurations and make sure to set hostname parameter.", $servers[$i]->{id}, ); $log->error($msg) if ($log); croak($msg); } } # check duplicate hosts for ( my $i = 0 ; $i <= $#servers ; $i++ ) { for ( my $j = $i + 1 ; $j <= $#servers ; $j++ ) { if ( $servers[$i]->{ip} eq $servers[$j]->{ip} && $servers[$i]->{port} eq $servers[$j]->{port} ) { my $msg = sprintf( "Server %s(hostname %s) and %s(hostname %s) have duplicate ip:port(%s:%d)! Check configurations.", $servers[$i]->{id}, $servers[$i]->{hostname}, $servers[$j]->{id}, $servers[$j]->{hostname}, $servers[$i]->{ip}, $servers[$i]->{port} ); $log->error($msg) if ($log); croak($msg); } } } return ( \@servers, \@binlog_servers ); } sub parse_server_default { my $self = shift; my $arg = shift; return $self->parse_server($arg); } sub print_msg { my $msg = shift; my $log = shift; if ($log) { $log->info($msg); } else { print "$msg\n"; } } sub add_block_and_save { my $file = shift; my $block_name = shift; my $hostname = shift; my $params_ref = shift; my $log = shift; my @params = @$params_ref; my $config = Config::Tiny->read($file); my $msg; unless ($config) { croak "Failed to open $file!"; } if ( $config->{$block_name} ) { croak "Entry $block_name already exists on $file ."; } $config->{$block_name}->{hostname} = $hostname; foreach my $param_value (@params) { my ( $key, $value ) = split( /=/, $param_value ); unless ( exists( $PARAM{$key} ) ) { croak "Parameter name $key is invalid!\n"; } else { $config->{$block_name}->{$key} = $value; } } $config->write($file); $msg = "Wrote $block_name entry to $file ."; print_msg( $msg, $log ); } sub delete_block_and_save { my $file = shift; my $block_name = shift; my $log = shift; my $config = Config::Tiny->read($file); my $msg; unless ($config) { $msg = "Failed to open $file!"; print_msg( $msg, $log ); return; } unless ( $config->{$block_name} ) { $msg = "Entry $block_name not found from $file ."; print_msg( $msg, $log ); return; } delete $config->{$block_name}; $config->write($file); $msg = "Deleted $block_name entry from $file ."; print_msg( $msg, $log ); } 1; mha4mysql-manager-0.58/lib/MHA/DBHelper.pm000066400000000000000000000651611325502577300201550ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::DBHelper; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use MHA::SlaveUtil; use MHA::ManagerConst; use Carp qw(croak); use DBI; use Data::Dumper; use Log::Dispatch; use constant Status => "Status"; use constant Errstr => "Errstr"; #show master status output use constant File => "File"; use constant Position => "Position"; use constant Binlog_Do_DB => "Binlog_Do_DB"; use constant Binlog_Ignore_DB => "Binlog_Ignore_DB"; use constant Executed_Gtid_Set => "Executed_Gtid_Set"; #show slave status output use constant Slave_IO_State => "Slave_IO_State"; use constant Slave_SQL_Running => "Slave_SQL_Running"; use constant Slave_IO_Running => "Slave_IO_Running"; use constant Master_Log_File => "Master_Log_File"; use constant Master_Host => "Master_Host"; use constant Master_User => "Master_User"; use constant Master_Port => "Master_Port"; use constant Replicate_Do_DB => "Replicate_Do_DB"; use constant Replicate_Ignore_DB => "Replicate_Ignore_DB"; use constant Replicate_Do_Table => "Replicate_Do_Table"; use constant Replicate_Ignore_Table => "Replicate_Ignore_Table"; use constant Replicate_Wild_Do_Table => "Replicate_Wild_Do_Table"; use constant Replicate_Wild_Ignore_Table => "Replicate_Wild_Ignore_Table"; use constant Read_Master_Log_Pos => "Read_Master_Log_Pos"; use constant Relay_Master_Log_File => "Relay_Master_Log_File"; use constant Exec_Master_Log_Pos => "Exec_Master_Log_Pos"; use constant Relay_Log_File => "Relay_Log_File"; use constant Relay_Log_Pos => "Relay_Log_Pos"; use constant Seconds_Behind_Master => "Seconds_Behind_Master"; use constant Last_Errno => "Last_Errno"; use constant Last_Error => "Last_Error"; use constant Retrieved_Gtid_Set => "Retrieved_Gtid_Set"; use constant Auto_Position => "Auto_Position"; use constant Set_Long_Wait_Timeout_SQL => "SET wait_timeout=86400"; use constant Show_One_Variable_SQL => "SHOW GLOBAL VARIABLES LIKE ?"; use constant Change_Master_SQL => "CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_USER='%s', MASTER_PASSWORD='%s', MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d"; use constant Change_Master_NoPass_SQL => "CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_USER='%s', MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d"; use constant Change_Master_Gtid_SQL => "CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_USER='%s', MASTER_PASSWORD='%s', MASTER_AUTO_POSITION=1"; use constant Change_Master_Gtid_NoPass_SQL => "CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_USER='%s', MASTER_AUTO_POSITION=1"; use constant Reset_Slave_Master_Host_SQL => "RESET SLAVE /*!50516 ALL */"; use constant Reset_Slave_SQL => "RESET SLAVE"; use constant Change_Master_Clear_SQL => "CHANGE MASTER TO MASTER_HOST=''"; use constant Show_Slave_Status_SQL => "SHOW SLAVE STATUS"; # i_s.processlist was not supported in older versions #use constant Show_Processlist_SQLThread_SQL=>"select * from information_schema.processlist where user='system user' and state like 'Has read all relay log%';"; use constant Show_Processlist_SQL => "SHOW PROCESSLIST"; use constant Show_Master_Status_SQL => "SHOW MASTER STATUS"; use constant Stop_IO_Thread_SQL => "STOP SLAVE IO_THREAD"; use constant Start_IO_Thread_SQL => "START SLAVE IO_THREAD"; use constant Start_Slave_SQL => "START SLAVE"; use constant Stop_Slave_SQL => "STOP SLAVE"; use constant Start_SQL_Thread_SQL => "START SLAVE SQL_THREAD"; use constant Stop_SQL_Thread_SQL => "STOP SLAVE SQL_THREAD"; use constant Get_Basedir_SQL => "SELECT \@\@global.basedir AS Value"; use constant Get_Datadir_SQL => "SELECT \@\@global.datadir AS Value"; use constant Get_Num_Workers_SQL => "SELECT \@\@global.slave_parallel_workers AS Value"; use constant Get_MaxAllowedPacket_SQL => "SELECT \@\@global.max_allowed_packet AS Value"; use constant Set_MaxAllowedPacket1G_SQL => "SET GLOBAL max_allowed_packet=1*1024*1024*1024"; use constant Set_MaxAllowedPacket_SQL => "SET GLOBAL max_allowed_packet=%d"; use constant Is_Readonly_SQL => "SELECT \@\@global.read_only As Value"; use constant Has_Gtid_SQL => "SELECT \@\@global.gtid_mode As Value"; use constant Get_ServerID_SQL => "SELECT \@\@global.server_id As Value"; use constant Unset_Readonly_SQL => "SET GLOBAL read_only=0"; use constant Set_Readonly_SQL => "SET GLOBAL read_only=1"; use constant Unset_Log_Bin_Local_SQL => "SET sql_log_bin=0"; use constant Set_Log_Bin_Local_SQL => "SET sql_log_bin=1"; use constant Rename_User_SQL => "RENAME USER '%s'\@'%%' TO '%s'\@'%%'"; use constant Master_Pos_Wait_NoTimeout_SQL => "SELECT MASTER_POS_WAIT(?,?,0) AS Result"; use constant Gtid_Wait_NoTimeout_SQL => "SELECT WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS(?,0) AS Result"; use constant Get_Connection_Id_SQL => "SELECT CONNECTION_ID() AS Value"; use constant Flush_Tables_Nolog_SQL => "FLUSH NO_WRITE_TO_BINLOG TABLES"; use constant Flush_Tables_With_Read_Lock_SQL => "FLUSH TABLES WITH READ LOCK"; use constant Unlock_Tables_SQL => "UNLOCK TABLES"; use constant Repl_User_SQL => "SELECT Repl_slave_priv AS Value FROM mysql.user WHERE user = ?"; use constant Select_User_Regexp_SQL => "SELECT user, host, password FROM mysql.user WHERE user REGEXP ? AND host REGEXP ?"; use constant Set_Password_SQL => "SET PASSWORD FOR ?\@? = ?"; use constant Old_Password_Length => 16; use constant Blocked_Empty_Password => '?' x 41; use constant Blocked_Old_Password_Head => '~' x 25; use constant Blocked_New_Password_Regexp => qr/^[0-9a-fA-F]{40}\*$/o; use constant Released_New_Password_Regexp => qr/^\*[0-9a-fA-F]{40}$/o; sub new { my $class = shift; my $self = { dsn => undef, dbh => undef, connection_id => undef, has_gtid => undef, is_mariadb => undef, @_, }; return bless $self, $class; } sub get_connection_id($) { my $self = shift; my $sth = $self->{dbh}->prepare(Get_Connection_Id_SQL); $sth->execute(); my $href = $sth->fetchrow_hashref; return $href->{Value}; } sub connect_util { my $host = shift; my $port = shift; my $user = shift; my $password = shift; my $dsn_host = $host =~ m{:} ? '[' . $host . ']' : $host; my $dsn = "DBI:mysql:;host=$dsn_host;port=$port;mysql_connect_timeout=1"; my $dbh = DBI->connect( $dsn, $user, $password, { PrintError => 0 } ); return $dbh; } sub check_connection_fast_util { my $host = shift; my $port = shift; my $user = shift; my $password = shift; my $dbh = connect_util( $host, $port, $user, $password ); if ( defined($dbh) ) { $dbh->disconnect(); return "1:Connection Succeeded"; } else { my $mysql_err = DBI->err; if ( $mysql_err && grep ( $_ == $mysql_err, @MHA::ManagerConst::ALIVE_ERROR_CODES ) > 0 ) { my $rc = $mysql_err; $rc .= "($DBI::errstr)" if ($DBI::errstr); return $rc; } } #server is dead return 0; } sub connect { my $self = shift; my $host = shift; my $port = shift; my $user = shift; my $password = shift; my $raise_error = shift; my $max_retries = shift; $raise_error = 0 if ( !defined($raise_error) ); $max_retries = 2 if ( !defined($max_retries) ); $self->{dbh} = undef; unless ( $self->{dsn} ) { my $dsn_host = $host =~ m{:} ? '[' . $host . ']' : $host; $self->{dsn} = "DBI:mysql:;host=$dsn_host;port=$port;mysql_connect_timeout=4"; } my $defaults = { PrintError => 0, RaiseError => ( $raise_error ? 1 : 0 ), }; while ( !$self->{dbh} && $max_retries-- ) { eval { $self->{dbh} = DBI->connect( $self->{dsn}, $user, $password, $defaults ); }; if ( !$self->{dbh} && $@ ) { croak $@ if ( !$max_retries ); } } if ( $self->{dbh} ) { $self->{connection_id} = $self->get_connection_id(); $self->{dbh}->{InactiveDestroy} = 1; } return $self->{dbh}; } sub disconnect($) { my $self = shift; $self->{dbh}->disconnect() if ( $self->{dbh} ); } sub get_variable { my $self = shift; my $query = shift; my $sth = $self->{dbh}->prepare($query); $sth->execute(); my $href = $sth->fetchrow_hashref; return $href->{Value}; } # display one value that are not supported by select @@.. sub show_variable($$) { my $self = shift; my $cond = shift; my $sth = $self->{dbh}->prepare(Show_One_Variable_SQL); $sth->execute($cond); my $href = $sth->fetchrow_hashref; return $href->{Value}; } sub has_repl_priv { my $self = shift; my $user = shift; my $sth = $self->{dbh}->prepare(Repl_User_SQL); my $ret = $sth->execute($user); if ( !defined($ret) ) { croak "Got MySQL error when checking replication privilege. $DBI::err: $DBI::errstr query:" . Repl_User_SQL . "\n"; } my $href = $sth->fetchrow_hashref; my $value = $href->{Value}; return 1 if ( defined($value) && $value eq "Y" ); return 0; } sub is_binlog_enabled($) { my $self = shift; my $value = $self->show_variable("log_bin"); return 1 if ( defined($value) && $value eq "ON" ); return 0; } sub is_read_only($) { my $self = shift; return $self->get_variable(Is_Readonly_SQL); } sub has_gtid($) { my $self = shift; my $value = $self->get_variable(Has_Gtid_SQL); if ( defined($value) && $value eq "ON" ) { $self->{has_gtid} = 1; return 1; } return 0; } sub get_basedir($) { my $self = shift; return $self->get_variable(Get_Basedir_SQL); } sub get_datadir($) { my $self = shift; return $self->get_variable(Get_Datadir_SQL); } sub get_num_workers($) { my $self = shift; return $self->get_variable(Get_Num_Workers_SQL); } sub get_version($) { my $self = shift; my $value = MHA::SlaveUtil::get_version( $self->{dbh} ); if ( $value =~ /MariaDB/ ) { $self->{is_mariadb} = 1; } return $value; } sub is_relay_log_purge($) { my $self = shift; return MHA::SlaveUtil::is_relay_log_purge( $self->{dbh} ); } sub disable_relay_log_purge($) { my $self = shift; return MHA::SlaveUtil::disable_relay_log_purge( $self->{dbh} ); } sub get_relay_log_info_type { my ( $self, $mysql_version ) = @_; return MHA::SlaveUtil::get_relay_log_info_type( $self->{dbh}, $mysql_version ); } sub get_relay_log_info_path { my ( $self, $mysql_version ) = @_; return MHA::SlaveUtil::get_relay_log_info_path( $self->{dbh}, $mysql_version ); } sub get_server_id($) { my $self = shift; return $self->get_variable(Get_ServerID_SQL); } sub get_max_allowed_packet($) { my $self = shift; return $self->get_variable(Get_MaxAllowedPacket_SQL); } sub set_max_allowed_packet($$) { my $self = shift; my $size = shift; my $query = sprintf( Set_MaxAllowedPacket_SQL, $size ); return $self->execute($query); } sub set_max_allowed_packet_1g($) { my $self = shift; return $self->execute(Set_MaxAllowedPacket1G_SQL); } sub show_master_status($) { my $self = shift; my ( $query, $sth, $href ); my %values; $query = Show_Master_Status_SQL; $sth = $self->{dbh}->prepare($query); my $ret = $sth->execute(); return if ( !defined($ret) || $ret != 1 ); $href = $sth->fetchrow_hashref; for my $key ( File, Position, Executed_Gtid_Set ) { $values{$key} = $href->{$key}; } for my $filter_key ( Binlog_Do_DB, Binlog_Ignore_DB ) { $values{$filter_key} = uniq_and_sort( $href->{$filter_key} ); } return ( $values{File}, $values{Position}, $values{Binlog_Do_DB}, $values{Binlog_Ignore_DB}, $values{Executed_Gtid_Set} ); } sub execute_update { my ( $self, $query, $expected_affected_rows, $bind_args ) = @_; my %status = (); my @params; if ( defined $bind_args ) { push @params, @$bind_args; } my $sth = $self->{dbh}->prepare($query); my $ret = $sth->execute(@params); if ( !defined($ret) || $ret != $expected_affected_rows ) { $status{Status} = -1; $status{Errstr} = $sth->errstr; return %status; } $status{Status} = $ret; return %status; } sub execute { my ( $self, $query, $bind_args ) = @_; my %status = $self->execute_update( $query, 0E0, $bind_args ); if ( $status{Status} == 0 ) { return 0; } elsif ( $status{Errstr} ) { return $status{Errstr}; } else { return 1; } } sub flush_tables_nolog($) { my $self = shift; return $self->execute(Flush_Tables_Nolog_SQL); } sub flush_tables_with_read_lock($) { my $self = shift; my $result = $self->execute(Set_Readonly_SQL); if ($result) { $result = $self->execute(Flush_Tables_With_Read_Lock_SQL); } return $result; } sub unlock_tables($) { my $self = shift; return $self->execute(Unlock_Tables_SQL); } sub set_wait_timeout_util($$) { my $dbh = shift; my $timeout = shift; my $sth = $dbh->prepare( sprintf( "SET wait_timeout=%d", $timeout ) ); my $ret = $sth->execute(); return 1 if ( !defined($ret) || $ret != 0E0 ); return 0; } sub set_long_wait_timeout($) { my $self = shift; return $self->execute(Set_Long_Wait_Timeout_SQL); } sub reset_slave_master_host($) { my $self = shift; return $self->execute(Reset_Slave_Master_Host_SQL); } sub reset_slave_by_change_master($) { my $self = shift; return $self->execute(Change_Master_Clear_SQL); } sub change_master($$$$$$$) { my $self = shift; my $master_host = shift; my $master_port = shift; my $master_log_file = shift; my $master_log_pos = shift; my $master_user = shift; my $master_password = shift; my $query; if ($master_password) { $query = sprintf( Change_Master_SQL, $master_host, $master_port, $master_user, $master_password, $master_log_file, $master_log_pos ); } else { $query = sprintf( Change_Master_NoPass_SQL, $master_host, $master_port, $master_user, $master_log_file, $master_log_pos ); } return $self->execute($query); } sub change_master_gtid($$$$$) { my $self = shift; my $master_host = shift; my $master_port = shift; my $master_user = shift; my $master_password = shift; my $query; if ($master_password) { $query = sprintf( Change_Master_Gtid_SQL, $master_host, $master_port, $master_user, $master_password ); } else { $query = sprintf( Change_Master_Gtid_NoPass_SQL, $master_host, $master_port, $master_user ); } return $self->execute($query); } sub disable_log_bin_local($) { my $self = shift; return $self->execute(Unset_Log_Bin_Local_SQL); } sub enable_log_bin_local($) { my $self = shift; return $self->execute(Set_Log_Bin_Local_SQL); } sub enable_read_only($) { my $self = shift; if ( $self->is_read_only() eq "1" ) { return 0; } else { return $self->execute(Set_Readonly_SQL); } } sub disable_read_only($) { my $self = shift; if ( $self->is_read_only() eq "0" ) { return 0; } else { return $self->execute(Unset_Readonly_SQL); } } sub reset_slave($) { my $self = shift; return $self->execute(Reset_Slave_SQL); } sub start_io_thread($) { my $self = shift; return $self->execute(Start_IO_Thread_SQL); } sub stop_io_thread($) { my $self = shift; return $self->execute(Stop_IO_Thread_SQL); } sub start_slave() { my $self = shift; return $self->execute(Start_Slave_SQL); } sub start_sql_thread() { my $self = shift; return $self->execute(Start_SQL_Thread_SQL); } sub stop_sql_thread() { my $self = shift; return $self->execute(Stop_SQL_Thread_SQL); } sub stop_slave() { my $self = shift; return $self->execute(Stop_Slave_SQL); } sub uniq_and_sort { my $str = shift; my @array = split( /,/, $str ); my %count; @array = grep( !$count{$_}++, @array ); @array = sort @array; return join( ',', @array ); } sub check_slave_status { my $self = shift; my $allow_dummy = shift; my ( $query, $sth, $href ); my %status = (); unless ( $self->{dbh} ) { $status{Status} = 1; $status{Errstr} = "Database Handle is not defined!"; return %status; } $query = Show_Slave_Status_SQL; $sth = $self->{dbh}->prepare($query); my $ret = $sth->execute(); if ( !defined($ret) || $ret != 1 ) { # I am not a slave $status{Status} = 1; # unexpected error if ( defined( $sth->errstr ) ) { $status{Status} = 2; $status{Errstr} = "Got error when executing " . Show_Slave_Status_SQL . ". " . $sth->errstr; } return %status; } $status{Status} = 0; $href = $sth->fetchrow_hashref; for my $key ( Slave_IO_State, Master_Host, Master_Port, Master_User, Slave_IO_Running, Slave_SQL_Running, Master_Log_File, Read_Master_Log_Pos, Relay_Master_Log_File, Last_Errno, Last_Error, Exec_Master_Log_Pos, Relay_Log_File, Relay_Log_Pos, Seconds_Behind_Master, Retrieved_Gtid_Set, Executed_Gtid_Set, Auto_Position ) { $status{$key} = $href->{$key}; } if ( !$status{Master_Host} || !$status{Master_Log_File} ) { unless ($allow_dummy) { # I am not a slave $status{Status} = 1; return %status; } } for my $filter_key ( Replicate_Do_DB, Replicate_Ignore_DB, Replicate_Do_Table, Replicate_Ignore_Table, Replicate_Wild_Do_Table, Replicate_Wild_Ignore_Table ) { $status{$filter_key} = uniq_and_sort( $href->{$filter_key} ); } return %status; } sub wait_until_relay_io_log_applied($$$) { my $self = shift; my $log = shift; my $num_worker_threads = shift; return read_all_relay_log( $self, $log, $num_worker_threads, 1, 1 ); } # wait until slave executes all relay logs. # MASTER_LOG_POS() must not be used sub wait_until_relay_log_applied($$$) { my $self = shift; my $log = shift; my $num_worker_threads = shift; return read_all_relay_log( $self, $log, $num_worker_threads, 1 ); } sub read_all_relay_log { my $self = shift; my $log = shift; my $num_worker_threads = shift; my $wait_until_latest = shift; my $io_thread_should_run = shift; $wait_until_latest = 0 if ( !defined($wait_until_latest) ); $io_thread_should_run = 0 if ( !defined($io_thread_should_run) ); my $sql_thread_check; my %status; do { $sql_thread_check = 1; %status = $self->check_slave_status(); if ( $status{Status} != 0 ) { return %status; } elsif ( !$io_thread_should_run && $status{Slave_IO_Running} eq "Yes" ) { $status{Status} = 3; $status{Errstr} = "Slave IO thread is running! Check master status."; return %status; } elsif ( $status{Slave_SQL_Running} eq "No" ) { $status{Status} = 4; $status{Errstr} = "SQL thread is not running! Check slave status."; return %status; } elsif ( ( $status{Master_Log_File} eq $status{Relay_Master_Log_File} ) && ( $status{Read_Master_Log_Pos} == $status{Exec_Master_Log_Pos} ) ) { $status{Status} = 0; return %status; } if ($io_thread_should_run) { if (!$status{Slave_IO_State} || $status{Slave_IO_State} !~ m/Waiting for master to send event/ ) { $sql_thread_check = 0; } } if ($sql_thread_check) { my $sql_thread_done = 0; my $worker_thread_done = 0; my $current_workers = 0; my $sth = $self->{dbh}->prepare(Show_Processlist_SQL); $sth->execute(); while ( my $ref = $sth->fetchrow_hashref ) { my $user = $ref->{User}; my $state = $ref->{State}; if ( defined($user) && $user eq "system user" && defined($state) ) { if ( $state =~ m/^Has read all relay log/ || $state =~ m/^Slave has read all relay log/ ) { $sql_thread_done = 1; if ( $num_worker_threads == 0 ) { $worker_thread_done = 1; } if ($worker_thread_done) { last; } } elsif ( $state =~ m/^Waiting for an event from Coordinator/ ) { $current_workers++; if ( $current_workers >= $num_worker_threads ) { $worker_thread_done = 1; } } } if ( $worker_thread_done == 1 && $sql_thread_done == 1 ) { last; } } if ( $sql_thread_done == 1 && $worker_thread_done == 1 ) { $status{Status} = 0; return %status; } $log->debug( sprintf( "Sql Thread Done: %d, Worker Thread done: %d, Ended workers: %d", $sql_thread_done, $worker_thread_done, $current_workers ) ); } } while ( $wait_until_latest && sleep(1) ); $status{Status} = 1; $status{Errstr} = "Unexpected error happened on waiting reading all relay logs."; return %status; } sub get_threads_util { my $dbh = shift; my $my_connection_id = shift; my $running_time_threshold = shift; my $type = shift; $running_time_threshold = 0 unless ($running_time_threshold); $type = 0 unless ($type); my @threads; my $sth = $dbh->prepare(Show_Processlist_SQL); $sth->execute(); while ( my $ref = $sth->fetchrow_hashref() ) { my $id = $ref->{Id}; my $user = $ref->{User}; my $host = $ref->{Host}; my $command = $ref->{Command}; my $state = $ref->{State}; my $query_time = $ref->{Time}; my $info = $ref->{Info}; $info =~ s/^\s*(.*?)\s*$/$1/ if defined($info); next if ( $my_connection_id == $id ); next if ( defined($query_time) && $query_time < $running_time_threshold ); next if ( defined($command) && $command =~ /^Binlog Dump/ ); next if ( defined($user) && $user eq "system user" ); next if ( defined($user) && $user eq "event_scheduler" ); if ( $type >= 1 ) { next if ( defined($command) && $command eq "Sleep" ); next if ( defined($command) && $command eq "Connect" ); } if ( $type >= 2 ) { next if ( defined($info) && $info =~ m/^select/i ); next if ( defined($info) && $info =~ m/^show/i ); } push @threads, $ref; } return @threads; } sub print_threads_util { my ( $threads_ref, $max_prints ) = @_; my @threads = @$threads_ref; my $count = 0; print "Details:\n"; foreach my $thread (@threads) { print Data::Dumper->new( [$thread] )->Indent(0)->Terse(1)->Dump . "\n"; $count++; if ( $count >= $max_prints ) { printf( "And more.. (%d threads in total)\n", $#threads + 1 ); last; } } } sub get_threads($$$) { my $self = shift; my $running_time_threshold = shift; my $type = shift; return MHA::DBHelper::get_threads_util( $self->{dbh}, $self->{connection_id}, $running_time_threshold, $type ); } sub get_running_threads($$) { my $self = shift; my $running_time_threshold = shift; return $self->get_threads( $running_time_threshold, 1 ); } sub get_running_update_threads($$) { my $self = shift; my $running_time_threshold = shift; return $self->get_threads( $running_time_threshold, 2 ); } sub kill_threads { my ( $self, @threads ) = @_; foreach (@threads) { kill_thread_util( $self->{dbh}, $_->{Id} ); } } sub kill_thread_util { my $dbh = shift; my $id = shift; eval { my $sth = $dbh->prepare("KILL ?"); $sth->execute($id); }; if ($@) { my $mysql_err = $dbh->err; if ( $mysql_err && $mysql_err == $MHA::ManagerConst::MYSQL_UNKNOWN_TID ) { $@ = undef; return; } croak $@; } } sub rename_user($$$) { my $self = shift; my $from_user = shift; my $to_user = shift; my $query = sprintf( Rename_User_SQL, $from_user, $to_user ); return $self->execute($query); } sub execute_ddl($$) { my ( $self, $query ) = @_; return $self->execute($query); } sub master_pos_wait($$$) { my $self = shift; my $binlog_file = shift; my $binlog_pos = shift; my $sth = $self->{dbh}->prepare(Master_Pos_Wait_NoTimeout_SQL); $sth->execute( $binlog_file, $binlog_pos ); my $href = $sth->fetchrow_hashref; return $href->{Result}; } sub gtid_wait($$) { my $self = shift; my $exec_gtid = shift; my $sth = $self->{dbh}->prepare(Gtid_Wait_NoTimeout_SQL); $sth->execute($exec_gtid); my $href = $sth->fetchrow_hashref; return $href->{Result}; } sub _blocked_password { my $password = shift; if ( $password eq '' ) { return Blocked_Empty_Password; } elsif ( length($password) == Old_Password_Length ) { return Blocked_Old_Password_Head . $password; } elsif ( $password =~ Released_New_Password_Regexp ) { return join( "", reverse( split //, $password ) ); } else { return; } } sub _released_password { my $password = shift; if ( $password eq Blocked_Empty_Password ) { return ''; } elsif ( index( $password, Blocked_Old_Password_Head ) == 0 ) { return substr( $password, length(Blocked_Old_Password_Head) ); } elsif ( $password =~ Blocked_New_Password_Regexp ) { return join( "", reverse( split //, $password ) ); } else { return; } } sub _block_release_user_by_regexp { my ( $dbh, $user, $host, $block ) = @_; my $users_to_block = $dbh->selectall_arrayref( Select_User_Regexp_SQL, { Slice => {} }, $user, $host ); my $failure = 0; for my $u ( @{$users_to_block} ) { my $password = $block ? _blocked_password( $u->{password} ) : _released_password( $u->{password} ); if ( defined $password ) { my $ret = $dbh->do( Set_Password_SQL, undef, $u->{user}, $u->{host}, $password ); unless ( $ret eq "0E0" ) { $failure++; } } } return $failure; } sub block_user_regexp { my ( $self, $user, $host ) = @_; return _block_release_user_by_regexp( $self->{dbh}, $user, $host, 1 ); } sub release_user_regexp { my ( $self, $user, $host ) = @_; return _block_release_user_by_regexp( $self->{dbh}, $user, $host, 0 ); } 1; mha4mysql-manager-0.58/lib/MHA/FileStatus.pm000066400000000000000000000060031325502577300206010ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::FileStatus; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Carp qw(croak); use File::Basename; use MHA::ManagerConst; use MHA::NodeUtil; sub new { my $class = shift; my $self = { conffile => undef, dir => undef, status_file => undef, basename => undef, master_host => undef, @_, }; return bless $self, $class; } sub init($) { my $self = shift; unless ( $self->{basename} ) { $self->{basename} = $self->get_basename(); } unless ( $self->{status_file} ) { $self->{status_file} = "$self->{dir}/" . $self->{basename} . ".master_status.health"; } } sub set_master_host($$) { my $self = shift; my $master_host = shift; $self->{master_host} = $master_host; } # utility function. getting basename from conf file sub get_basename($) { my $self = shift; my $basename = basename( $self->{conffile} ); $basename =~ s/(.*)\..*?$/$1/; if ( $basename eq '' ) { $basename = $self->{conffile}; } return $basename; } sub update_status { my $self = shift; my $status_string = shift; my $file = $self->{status_file}; my $master_host = $self->{master_host}; my $out; if ( -f $file ) { open( $out, "+<", $file ) or croak "$!:$file"; flock( $out, 2 ); truncate( $out, 0 ); seek( $out, 0, 0 ); } else { open( $out, ">", $file ) or croak "$!:$file"; } print $out "$$\t$status_string"; print $out "\tmaster:$master_host" if ($master_host); close($out); } sub read_status { my $self = shift; my $file = $self->{status_file}; open( my $in, "+<", $file ) or croak "$!:$file"; flock( $in, 2 ); my $line = readline($in); my @values = split( /\t/, $line ); my $pid = $values[0]; my $status_string = $values[1]; my $master_info; if ( $values[2] ) { $master_info = $values[2]; } close($in); return ( $pid, $status_string, $master_info ); } sub update_status_time { my $self = shift; my $status_string = shift; my $file = $self->{status_file}; my $master_host = $self->{master_host}; if ( !-f $file ) { $self->update_status($status_string); } else { my $ftime = time; utime $ftime, $ftime, $file; } } 1; mha4mysql-manager-0.58/lib/MHA/HealthCheck.pm000066400000000000000000000477201325502577300206740ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::HealthCheck; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use English qw(-no_match_vars); use Time::HiRes qw( sleep gettimeofday tv_interval ); use POSIX; use DBI; use IO::File; use MHA::DBHelper; use MHA::ManagerConst; use MHA::FileStatus; use MHA::SlaveUtil; use MHA::NodeUtil; sub new { my $class = shift; my $self = { dbh => undef, interval => undef, user => undef, password => undef, ip => undef, hostname => undef, port => undef, ssh_user => undef, ssh_host => undef, ssh_ip => undef, ssh_port => undef, ssh_check_command => undef, ssh_connection_timeout => undef, workdir => undef, status_handler => undef, secondary_check_script => undef, logger => undef, logfile => undef, ping_type => undef, # internal (read/write) variables _tstart => undef, _already_monitored => 0, _need_reconnect => 1, _last_ping_fail => 1, _sec_check_invoked => 0, _sec_check_pid => undef, _ssh_check_invoked => 0, _ssh_check_pid => undef, @_, }; return bless $self, $class; } sub connect { my $self = shift; my $connect_timeout = shift; my $wait_timeout = shift; my $advisory_lock_timeout = shift; my $log_connect_error = shift; my $raise_error = shift; my $no_advisory_lock = shift; if ( !defined($connect_timeout) ) { $connect_timeout = $self->{interval}; } if ( !defined($wait_timeout) ) { $wait_timeout = $connect_timeout * 2; } if ( !defined($advisory_lock_timeout) ) { $advisory_lock_timeout = $wait_timeout * 2; } if ( !defined($log_connect_error) ) { $log_connect_error = 1; } if ( !defined($raise_error) ) { $raise_error = 0; } my $log = $self->{logger}; my $dsn_host = $self->{ip} =~ m{:} ? '[' . $self->{ip} . ']' : $self->{ip}; $self->{dbh} = DBI->connect( "DBI:mysql:;host=$dsn_host;" . "port=$self->{port};mysql_connect_timeout=$connect_timeout", $self->{user}, $self->{password}, { PrintError => 0, RaiseError => $raise_error } ); if ( $self->{dbh} ) { $log->debug("Connected on master."); $self->{dbh}->{InactiveDestroy} = 1; $self->set_wait_timeout($wait_timeout); my $rc = 0; unless ($no_advisory_lock) { $log->debug("Trying to get advisory lock.."); $rc = MHA::SlaveUtil::get_monitor_advisory_lock( $self->{dbh}, $advisory_lock_timeout ); } if ( $rc == 0 ) { if ( $self->{ping_type} eq $MHA::ManagerConst::PING_TYPE_INSERT ) { my $child_exit_code; eval { $child_exit_code = $self->fork_exec( sub { $self->ping_insert() }, "MySQL Ping($self->{ping_type})" ); }; if ($@) { my $msg = "Unexpected error heppened when pinging! $@"; $log->error($msg); undef $@; $child_exit_code = 1; } return $child_exit_code; } return 0; } elsif ( $rc == 1 ) { # locked by someone or (in rare cases) my previous uncleaned connection $self->{_already_monitored} = 1; croak; } else { my $msg = "Got unexpected error on getting MySQL advisory lock: "; $msg .= $DBI::err if ($DBI::err); $msg .= " ($DBI::errstr)" if ($DBI::errstr); $log->warning($msg); return 1; } } else { my $msg = "Got error on MySQL connect: "; $msg .= $DBI::err if ($DBI::err); $msg .= " ($DBI::errstr)" if ($DBI::errstr); if ($log_connect_error) { $log->warning($msg); } else { $log->debug($msg); } return ( 1, $DBI::err ); } } sub disconnect_if { my $self = shift; my $dbh = $self->{dbh}; $dbh->disconnect() if ($dbh); $self->{dbh} = undef; } sub set_ping_interval($$) { my $self = shift; my $interval = shift; $self->{interval} = $interval if ($interval); return; } sub get_ping_interval($) { my $self = shift; return $self->{interval}; } sub set_secondary_check_script($$) { my $self = shift; my $script = shift; $self->{secondary_check_script} = $script if ($script); return; } sub get_secondary_check_script($) { my $self = shift; return $self->{secondary_check_script}; } sub set_ssh_user($$) { my $self = shift; my $ssh_user = shift; $self->{ssh_user} = $ssh_user if ($ssh_user); return; } sub get_ssh_user($) { my $self = shift; return $self->{ssh_user}; } sub set_workdir($$) { my $self = shift; my $workdir = shift; $self->{workdir} = $workdir if ($workdir); return; } sub get_workdir($) { my $self = shift; return $self->{workdir}; } sub set_wait_timeout($$) { my $self = shift; my $timeout = shift; my $log = $self->{logger}; my $dbh = $self->{dbh}; if ( MHA::DBHelper::set_wait_timeout_util( $self->{dbh}, $timeout ) ) { my $msg = "Got error on setting wait_timeout : $@ :"; $msg .= $DBI::err if ($DBI::err); $msg .= " ($DBI::errstr)" if ($DBI::errstr); $log->warning($msg); } else { $log->debug("Set short wait_timeout on master: $timeout seconds"); } } sub ping_connect($) { my $self = shift; my $log = $self->{logger}; my $dbh; my $rc = 1; my $max_retries = 2; eval { my $ping_start = [gettimeofday]; while ( !$self->{dbh} && $max_retries-- ) { eval { $rc = $self->connect( 1, $self->{interval}, 0, 0, 1 ); }; if ( !$self->{dbh} && $@ ) { die $@ if ( !$max_retries ); } } $rc = $self->ping_select(); # To hold advisory lock for some periods of time $self->sleep_until( $ping_start, $self->{interval} - 1.5 ); $self->disconnect_if(); }; if ($@) { my $msg = "Got error on MySQL connect ping: $@"; undef $@; $msg .= $DBI::err if ($DBI::err); $msg .= " ($DBI::errstr)" if ($DBI::errstr); $log->warning($msg) if ($log); $rc = 1; } return 2 if ( $self->{_already_monitored} ); return $rc; } sub ping_select($) { my $self = shift; my $log = $self->{logger}; my $dbh = $self->{dbh}; my ( $query, $sth, $href ); eval { $dbh->{RaiseError} = 1; $sth = $dbh->prepare("SELECT 1 As Value"); $sth->execute(); $href = $sth->fetchrow_hashref; if ( !defined($href) || !defined( $href->{Value} ) || $href->{Value} != 1 ) { die; } }; if ($@) { my $msg = "Got error on MySQL select ping: "; undef $@; $msg .= $DBI::err if ($DBI::err); $msg .= " ($DBI::errstr)" if ($DBI::errstr); $log->warning($msg) if ($log); return 1; } return 0; } sub ping_insert($) { my $self = shift; my $log = $self->{logger}; my $dbh = $self->{dbh}; my ( $query, $sth, $href ); eval { $dbh->{RaiseError} = 1; $dbh->do("CREATE DATABASE IF NOT EXISTS infra"); $dbh->do( "CREATE TABLE IF NOT EXISTS infra.chk_masterha (`key` tinyint NOT NULL primary key,`val` int(10) unsigned NOT NULL DEFAULT '0')" ); $dbh->do( "INSERT INTO infra.chk_masterha values (1,unix_timestamp()) ON DUPLICATE KEY UPDATE val=unix_timestamp()" ); }; if ($@) { my $msg = "Got error on MySQL insert ping: "; undef $@; $msg .= $DBI::err if ($DBI::err); $msg .= " ($DBI::errstr)" if ($DBI::errstr); $log->warning($msg) if ($log); return 1; } return 0; } sub ssh_check_simple { my $ssh_user = shift; my $ssh_host = shift; my $ssh_ip = shift; my $ssh_port = shift; my $log = shift; my $num_secs_to_timeout = shift; return ssh_check( $ssh_user, $ssh_host, $ssh_ip, $ssh_port, $log, $num_secs_to_timeout, "exit 0" ); } sub ssh_check { my $ssh_user = shift; my $ssh_host = shift; my $ssh_ip = shift; my $ssh_port = shift; my $log = shift; my $num_secs_to_timeout = shift; my $command = shift; my $ssh_user_host = $ssh_user . '@' . $ssh_ip; my $rc = 1; eval { if ( my $pid = fork ) { local $SIG{ALRM} = sub { kill 9, $pid; waitpid( $pid, 0 ); die "Got timeout on checking SSH connection to $ssh_host!"; }; $log->debug( "SSH connection test to $ssh_host, option $MHA::ManagerConst::SSH_OPT_CHECK, timeout $num_secs_to_timeout" ); alarm $num_secs_to_timeout; waitpid( $pid, 0 ); alarm 0; my ( $high, $low ) = MHA::NodeUtil::system_rc($?); if ( $high ne '0' || $low ne '0' ) { $log->warning("HealthCheck: SSH to $ssh_host is NOT reachable."); $rc = 1; } else { $log->info("HealthCheck: SSH to $ssh_host is reachable."); $rc = 0; } } elsif ( defined $pid ) { exec( "ssh $MHA::ManagerConst::SSH_OPT_CHECK -p $ssh_port $ssh_user_host \"$command\"" ); } else { croak "Forking SSH connection process failed!\n"; } }; alarm 0; if ($@) { $log->warning("HealthCheck: $@"); $rc = 1; } return $rc; } sub secondary_check($) { my $self = shift; my $log = $self->{logger}; my $command = "$self->{secondary_check_script} " . " --user=$self->{ssh_user} " . " --master_host=$self->{hostname} " . " --master_ip=$self->{ip} " . " --master_port=$self->{port}" . " --master_user=$self->{user}" . " --master_password=$self->{password}" . " --ping_type=$self->{ping_type}"; if ($MHA::ManagerConst::USE_SSH_OPTIONS) { $command .= " --options='$MHA::ManagerConst::SSH_OPT_CHECK' "; } $log->info("Executing secondary network check script: $command"); my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $self->{logfile} ); if ( $high == 0 && $low == 0 ) { $log->info( "Master is not reachable from all other monitoring " . "servers. Failover should start." ); return 0; } if ( $high == 2 ) { $log->warning( "At least one of monitoring servers is not reachable " . "from this script. This is likely a network problem. Failover should " . "not happen." ); return $high; } elsif ( $high == 3 ) { $log->warning( "Master is reachable from at least one of other " . "monitoring servers. Failover should not happen." ); return $high; } else { $log->error("Got unknown error from $command. exit."); return 1; } } sub terminate_child { my $self = shift; my $pid = shift; my $type = shift; my $num_secs_to_timeout = shift; unless ($num_secs_to_timeout) { $num_secs_to_timeout = $self->{interval}; } my $log = $self->{logger}; my $child_exit_code = 0; eval { local $SIG{ALRM} = sub { kill 9, $pid; waitpid( $pid, 0 ); die "Got timeout on $type child process and killed it!"; }; alarm $num_secs_to_timeout; waitpid( $pid, 0 ); $child_exit_code = $? >> 8; alarm 0; }; alarm 0; if ($@) { $log->warning($@) if ($log); undef $@; $child_exit_code = 1; } return $child_exit_code; } sub invoke_sec_check { my $self = shift; if ( !$self->{_sec_check_invoked} ) { if ( $self->{_sec_check_pid} = fork ) { $self->{_sec_check_invoked} = 1; } elsif ( defined $self->{_sec_check_pid} ) { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; #child secondary check process exit $self->secondary_check() if ( $self->{secondary_check_script} ); exit 0; } else { croak "Forking secondary check process failed. Can't continue operation.\n"; } } } sub invoke_ssh_check { my $self = shift; my $log = $self->{logger}; if ( !$self->{_ssh_check_invoked} ) { if ( $self->{_ssh_check_pid} = fork ) { $self->{_ssh_check_invoked} = 1; } elsif ( defined $self->{_ssh_check_pid} ) { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; $log->info("Executing SSH check script: $self->{ssh_check_command}"); #child ssh check process exit ssh_check( $self->{ssh_user}, $self->{ssh_host}, $self->{ssh_ip}, $self->{ssh_port}, $self->{logger}, $self->{ssh_connection_timeout}, $self->{ssh_check_command} ); } else { croak "Forking SSH check process failed. Can't continue operation.\n"; } } } sub is_secondary_down { my $self = shift; my $log = $self->{logger}; my $master_is_down = 0; eval { if ( $self->{_sec_check_invoked} ) { waitpid( $self->{_sec_check_pid}, 0 ); my $sec_check_exit_code = $?; $self->{_sec_check_invoked} = 0; if ( $sec_check_exit_code == 0 ) { $master_is_down = 1; } else { $log->warning( "Secondary network check script returned errors. Failover should not start so checking server status again. Check network settings for details." ); } } else { $master_is_down = 1; } }; if ($@) { $log->error("Got unexpected error on secondary network check: $@"); undef $@; } return $master_is_down; } sub is_ssh_reachable { my $self = shift; my $log = $self->{logger}; my $ssh_reachable = 2; eval { if ( $self->{_ssh_check_invoked} ) { waitpid( $self->{_ssh_check_pid}, 0 ); my $ssh_check_exit_code = $?; $self->{_ssh_check_invoked} = 0; if ( $ssh_check_exit_code == 0 ) { $ssh_reachable = 1; } else { $ssh_reachable = 0; } } }; if ($@) { $log->error("Got unexpected error on SSH check: $@"); undef $@; } return $ssh_reachable; } sub kill_sec_check { my $self = shift; my $num_secs_to_timeout = shift; my $exit_code = 1; if ( $self->{_sec_check_invoked} ) { if ( defined( $self->{_sec_check_pid} ) ) { $exit_code = $self->terminate_child( $self->{_sec_check_pid}, "Secondary Check", $num_secs_to_timeout ); } $self->{_sec_check_invoked} = 0; } return $exit_code; } sub kill_ssh_check { my $self = shift; my $num_secs_to_timeout = shift; my $exit_code = 1; if ( $self->{_ssh_check_invoked} ) { if ( defined( $self->{_ssh_check_pid} ) ) { $exit_code = $self->terminate_child( $self->{_ssh_check_pid}, "SSH Check", $num_secs_to_timeout ); } $self->{_ssh_check_invoked} = 0; } return $exit_code; } sub update_status_ok { my $self = shift; #updating status time filestamp if ( $self->{_last_ping_fail} ) { $self->{status_handler}->update_status($MHA::ManagerConst::ST_RUNNING_S); $self->{_last_ping_fail} = 0; } else { $self->{status_handler} ->update_status_time($MHA::ManagerConst::ST_RUNNING_S); } } sub sleep_until { my $self = shift; my $start = shift; my $interval = shift; unless ($start) { $start = $self->{_tstart}; } if ( !defined($interval) ) { $interval = $self->{interval}; } my $elapsed = tv_interval($start); if ( $interval > $elapsed ) { sleep( $interval - $elapsed ); } } sub handle_failing { my $self = shift; $self->{_last_ping_fail} = 1; $self->{status_handler}->update_status($MHA::ManagerConst::ST_PING_FAILING_S); $self->invoke_sec_check(); $self->invoke_ssh_check(); } sub fork_exec($$$) { my $self = shift; my $func = shift; my $type = shift; if ( my $pid = fork ) { return $self->terminate_child( $pid, $type ); } elsif ( defined $pid ) { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; exit $func->(); } else { croak "fork failed!\n"; } } # main function sub wait_until_unreachable($) { my $self = shift; my $log = $self->{logger}; my $ssh_reachable = 2; my $error_count = 0; my $master_is_down = 0; eval { while (1) { $self->{_tstart} = [gettimeofday]; if ( $self->{_need_reconnect} ) { my ( $rc, $mysql_err ) = $self->connect( undef, undef, undef, undef, undef, $error_count ); if ($rc) { if ($mysql_err) { if ( grep ( $_ == $mysql_err, @MHA::ManagerConst::ALIVE_ERROR_CODES ) > 0 ) { $log->info( "Got MySQL error $mysql_err, but this is not a MySQL crash. Continue health check.." ); $self->sleep_until(); next; } } $error_count++; $log->warning("Connection failed $error_count time(s).."); $self->handle_failing(); if ( $error_count >= 4 ) { $ssh_reachable = $self->is_ssh_reachable(); $master_is_down = 1 if ( $self->is_secondary_down() ); last if ($master_is_down); $error_count = 0; } $self->sleep_until(); next; } # connection ok $self->{_need_reconnect} = 0; $log->info( "Ping($self->{ping_type}) succeeded, waiting until MySQL doesn't respond.." ); } $self->disconnect_if() if ( $self->{ping_type} eq $MHA::ManagerConst::PING_TYPE_CONNECT ); # Parent process forks one child process. The child process queries # from MySQL every seconds. The child process may hang on # executing queries. # DBD::mysql 4.022 or earlier does not have an option to set # read timeout, executing queries might take forever. To avoid this, # the parent process kills the child process if it won't exit within # seconds. my $child_exit_code; eval { if ( $self->{ping_type} eq $MHA::ManagerConst::PING_TYPE_CONNECT ) { $child_exit_code = $self->fork_exec( sub { $self->ping_connect() }, "MySQL Ping($self->{ping_type})" ); } elsif ( $self->{ping_type} eq $MHA::ManagerConst::PING_TYPE_SELECT ) { $child_exit_code = $self->fork_exec( sub { $self->ping_select() }, "MySQL Ping($self->{ping_type})" ); } elsif ( $self->{ping_type} eq $MHA::ManagerConst::PING_TYPE_INSERT ) { $child_exit_code = $self->fork_exec( sub { $self->ping_insert() }, "MySQL Ping($self->{ping_type})" ); } else { die "Not supported ping_type!\n"; } }; if ($@) { my $msg = "Unexpected error heppened when pinging! $@"; $log->error($msg); undef $@; $child_exit_code = 1; } if ( $child_exit_code == 0 ) { #ping ok $self->update_status_ok(); if ( $error_count > 0 ) { $error_count = 0; } $self->kill_sec_check(); $self->kill_ssh_check(); } elsif ( $child_exit_code == 2 ) { $self->{_already_monitored} = 1; croak; } else { # failed on fork_exec $error_count++; $self->{_need_reconnect} = 1; $self->handle_failing(); } $self->sleep_until(); } $log->warning("Master is not reachable from health checker!"); }; if ($@) { my $msg = "Got error when monitoring master: $@"; $log->warning($msg); undef $@; return 2 if ( $self->{_already_monitored} ); return 1; } return 1 unless ($master_is_down); return ( 0, $ssh_reachable ); } 1; mha4mysql-manager-0.58/lib/MHA/ManagerAdmin.pm000066400000000000000000000245001325502577300210430ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::ManagerAdmin; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use File::Copy; use Errno (); use MHA::ManagerConst; use MHA::FileStatus; use MHA::ManagerUtil; use Time::HiRes qw( sleep ); $| = 1; sub new { my $class = shift; my $self = { app => undef, pid => undef, global_conf => $MHA::ManagerConst::DEFAULT_GLOBAL_CONF, conf => undef, status_dir => undef, status_file => undef, logfile => undef, master_info => undef, ping_limit => 10, status_handler => undef, abort => 0, abort_timeout => 5, @_, }; return bless $self, $class; } sub check_timestamp($) { my $self = shift; my $status_file = $self->{status_handler}->{status_file}; my $current_time = time(); my $ping_time = ( stat $status_file )[9]; unless ($ping_time) { print "Failed to get timestamp on $status_file!\n"; } if ( $current_time > $ping_time + $self->{ping_limit} ) { printf( "$status_file behinds %d seconds!\n", $current_time - $ping_time ); return 1; } return 0; } sub pid_trim($) { my @pids = shift; my @target = (); foreach my $pid (@pids) { next unless ($pid); chomp($pid); push @target, $pid if ( $pid > 1 ); } return @target; } sub get_child_pids($) { my $self = shift; my $pid = $self->{pid}; my @child_pids = (); @child_pids = `ps --ppid $pid | grep -v PID | awk '{print \$1}'`; return pid_trim(@child_pids); } sub is_pid_alive($) { my $self = shift; my $pid = $self->{pid}; my $pid_exists = kill 0, $pid; return $pid_exists; } sub check_status_file_pid($) { my $self = shift; if ( !-f $self->{status_handler}->{status_file} ) { return $MHA::ManagerConst::ST_NOT_RUNNING; } my $status_string; ( $self->{pid}, $status_string, $self->{master_info} ) = $self->{status_handler}->read_status(); my $pid_exists = $self->is_pid_alive(); unless ($pid_exists) { my @child_pids = $self->get_child_pids(); return $MHA::ManagerConst::ST_NOT_RUNNING if ( $#child_pids == -1 ); return $MHA::ManagerConst::ST_PARTIALLY_RUNNING; } return $MHA::ManagerConst::ST_INITIALIZING_MONITOR if ( $MHA::ManagerConst::ST_INITIALIZING_MONITOR_S eq $status_string ); return $MHA::ManagerConst::ST_RETRYING_MONITOR if ( $MHA::ManagerConst::ST_RETRYING_MONITOR_S eq $status_string ); return $MHA::ManagerConst::ST_PING_FAILING if ( $MHA::ManagerConst::ST_PING_FAILING_S eq $status_string ); return $MHA::ManagerConst::ST_PING_FAILED if ( $MHA::ManagerConst::ST_PING_FAILED_S eq $status_string ); return $MHA::ManagerConst::ST_CONFIG_ERROR if ( $MHA::ManagerConst::ST_CONFIG_ERROR_S eq $status_string ); return $MHA::ManagerConst::ST_FAILOVER_RUNNING if ( $MHA::ManagerConst::ST_FAILOVER_RUNNING_S eq $status_string ); return $MHA::ManagerConst::ST_FAILOVER_ERROR if ( $MHA::ManagerConst::ST_FAILOVER_ERROR_S eq $status_string ); if ( $MHA::ManagerConst::ST_RUNNING_S eq $status_string ) { if ( $self->check_timestamp() ) { return $MHA::ManagerConst::ST_TIMESTAMP_OLD; } else { return $MHA::ManagerConst::ST_RUNNING; } } croak "Got unknown status: $status_string\n"; } sub read_conf($) { my $self = shift; if ( !-f $self->{conf} ) { croak "$self->{conf} not found!\n"; } # Loading Config takes time so we load module only when really needed require MHA::Config; my $conf = new MHA::Config( globalfile => $self->{global_conf}, file => $self->{conf}, ); my ( $sc_ref, undef ) = $conf->read_config(); my @sc = @$sc_ref; $self->{status_dir} = $sc[0]->{manager_workdir} unless ( $self->{status_dir} ); $self->{logfile} = $sc[0]->{manager_log}; return $self; } sub init { my $self = shift; if ( $self->{conf} ) { $self = $self->read_conf(); } unless ( $self->{status_dir} ) { if ( $self->{conf} ) { croak "Failed to get status file directory (set in workdir parameter) from $self->{conf}.\n"; } else { croak "Either --conf or --status_dir must be set.\n"; } } if ( !-d $self->{status_dir} ) { croak "Directory $self->{status_dir} not found!\n"; } if ( $self->{conf} ) { $self->{status_handler} = new MHA::FileStatus( conffile => $self->{conf}, dir => $self->{status_dir} ); $self->{status_handler}->init(); $self->{app} = $self->{status_handler}->{basename} unless ( $self->{app} ); } unless ( $self->{app} ) { croak "app is not defined or could not be fetched from conf file!\n"; } unless ( $self->{conf} ) { $self->{status_handler} = new MHA::FileStatus( basename => $self->{app}, dir => $self->{status_dir} ); $self->{status_handler}->{basename} = $self->{app}; $self->{status_handler}->init(); } } sub stop_manager { my $self = shift; my $app; my $exit_code = 1; my $ret; $exit_code = eval { $self->init(); $app = $self->{app}; $ret = $self->check_status_file_pid(); if ( $ret == $MHA::ManagerConst::ST_NOT_RUNNING ) { print "MHA Manager is not running on $self->{app}($MHA::ManagerConst::ST_NOT_RUNNING_S).\n"; return 0; } elsif ( $ret == $MHA::ManagerConst::ST_FAILOVER_RUNNING && !$self->{abort} ) { print "Currently Failover is running on $self->{app}. Should not stop.\n"; return 1; } else { if ( kill -15, getpgrp $self->{pid} ) { my $millis_until_abort = $self->{abort_timeout} * 1000; while ( $millis_until_abort > 0 ) { my $pid_exists = kill 0, $self->{pid}; unless ($pid_exists) { print "Stopped $self->{app} successfully.\n"; return 0; } sleep(0.1); $millis_until_abort = $millis_until_abort - 100; } print "Manager process $self->{pid} didn't stop in $self->{abort_timeout} seconds.\n"; if ( $self->{abort} && ( kill -9, getpgrp $self->{pid} ) ) { print "Forced stopped $self->{app}.\n"; return 0; } } } print "Failed to stop $self->{app}.\n"; return 1; }; if ($@) { my $msg; if ( $self->{conf} ) { $msg .= "Got error on conf $self->{conf}: "; } elsif ( $self->{app} ) { $msg .= "Got error on app $app: "; } $msg .= $@; warn $msg; undef $@; } return 1 if ( !defined($exit_code) ); return $exit_code; } sub check_status { my $self = shift; my $app; my $ret = 1; eval { $self->init(); $app = $self->{app}; $ret = $self->check_status_file_pid(); if ( $ret == $MHA::ManagerConst::ST_RUNNING ) { print "$app (pid:$self->{pid}) is running($MHA::ManagerConst::ST_RUNNING_S), $self->{master_info}"; print "\n"; } elsif ( $ret == $MHA::ManagerConst::ST_NOT_RUNNING ) { print "$app is stopped($MHA::ManagerConst::ST_NOT_RUNNING_S).\n"; } elsif ( $ret == $MHA::ManagerConst::ST_PARTIALLY_RUNNING ) { my (@child_pids) = $self->get_child_pids(); print "Main process is not running, but child process is running on $app "; if ( $#child_pids > -1 ) { printf( "(child pid: %s)", join( ' ', @child_pids ) ); } print " ($MHA::ManagerConst::ST_PARTIALLY_RUNNING_S). Check ps output for details and kill it.\n"; } elsif ( $ret == $MHA::ManagerConst::ST_INITIALIZING_MONITOR ) { print "$app monitoring program is now on initialization phase($MHA::ManagerConst::ST_INITIALIZING_MONITOR_S). Wait for a while and try checking again.\n"; } elsif ( $ret == $MHA::ManagerConst::ST_PING_FAILING || $ret == $MHA::ManagerConst::ST_PING_FAILED ) { print "$app master maybe down"; print "($MHA::ManagerConst::ST_PING_FAILING_S)" if ( $ret == $MHA::ManagerConst::ST_PING_FAILING ); print "($MHA::ManagerConst::ST_PING_FAILED_S)" if ( $ret == $MHA::ManagerConst::ST_PING_FAILED ); print ". $self->{master_info}\n"; print "Check $self->{logfile} for details.\n" if ( $self->{logfile} ); } elsif ( $ret == $MHA::ManagerConst::ST_RETRYING_MONITOR ) { print "$app monitoring waits for retrying to monitor master again($MHA::ManagerConst::ST_RETRYING_MONITOR_S). Wait for a while and try checking again."; } elsif ( $ret == $MHA::ManagerConst::ST_CONFIG_ERROR ) { print "$app servers are not correctly configured($MHA::ManagerConst::ST_CONFIG_ERROR_S).\n"; print "Check $self->{logfile} for details.\n" if ( $self->{logfile} ); } elsif ( $ret == $MHA::ManagerConst::ST_TIMESTAMP_OLD ) { print "$app is running, but master_ping.health is too old($MHA::ManagerConst::ST_TIMESTAMP_OLD_S). Maybe process hangs?"; print " $self->{master_info}" if ( $self->{master_info} ); print "\n"; } elsif ( $ret == $MHA::ManagerConst::ST_FAILOVER_RUNNING ) { print "$app master is down and failover is running($MHA::ManagerConst::ST_FAILOVER_RUNNING_S). $self->{master_info}\n"; print "Check $self->{logfile} for details.\n" if ( $self->{logfile} ); } elsif ( $ret == $MHA::ManagerConst::ST_FAILOVER_ERROR ) { print "$app master is down and failover was not successful($MHA::ManagerConst::ST_FAILOVER_ERROR_S). $self->{master_info}\n"; print "Check $self->{logfile} for details.\n" if ( $self->{logfile} ); } else { print "Unexpected error on $app.\n"; } }; if ($@) { my $msg; if ( $self->{conf} ) { $msg .= "Got error on conf $self->{conf}: "; } elsif ( $self->{app} ) { $msg .= "Got error on app $app: "; } $msg .= $@; warn $msg; undef $@; } return $ret; } 1; mha4mysql-manager-0.58/lib/MHA/ManagerAdminWrapper.pm000066400000000000000000000071371325502577300224130ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::ManagerAdminWrapper; use strict; use warnings FATAL => 'all'; use Getopt::Long qw(:config pass_through); use MHA::ManagerConst; use MHA::ManagerAdmin; my $default_confdir = "/usr/local/masterha/conf"; my $all; my $app; my $global_conf = $MHA::ManagerConst::DEFAULT_GLOBAL_CONF; my $conf; my $basedir; my $baselog; my $internal_basedir; my $status_dir; my $logfile; my $ping_limit = 10; my $abort; my $abort_timeout = 5; sub get_masterha_daemontool_appnames { opendir my $dir, "/service"; my @dirs = map { my $s = $_; $s =~ s/^masterha_//; $s } grep { m/^masterha_/ } readdir $dir; @dirs = sort @dirs; closedir $dir; return @dirs; } sub process_all { my $command = shift; my @apps = get_masterha_daemontool_appnames(); my $ret = 0; foreach my $appname (@apps) { $conf = "$default_confdir/$appname.cnf"; if ($basedir) { $status_dir = "$basedir/$appname"; $app = $appname; undef $conf; if ($baselog) { $logfile = "$baselog/$appname/$appname.log"; } } my $app_ret; $app_ret = check_single_app_status() if ( $command == 1 ); $app_ret = stop_single_app_manager() if ( $command == 2 ); $ret = 1 if ($app_ret); } return $ret; } sub init { GetOptions( 'all' => \$all, 'app=s' => \$app, 'global_conf=s' => \$global_conf, 'conf=s' => \$conf, 'basedir=s' => \$basedir, 'baselog=s' => \$baselog, 'status_dir=s' => \$status_dir, 'manager_workdir=s' => \$status_dir, 'workdir=s' => \$status_dir, 'log_output=s' => \$logfile, 'manager_log=s' => \$logfile, 'ping_limit=i' => \$ping_limit, 'abort' => \$abort, 'abort_timeout=i' => \$abort_timeout, ); } sub check_status { init(); if ($all) { return process_all(1); } else { if ( $app && !$conf && !$status_dir ) { $conf = "$default_confdir/$app.cnf"; } return check_single_app_status(); } } sub stop_manager { init(); if ($all) { return process_all(2); } else { if ( $app && !$conf && !$status_dir ) { $conf = "$default_confdir/$app.cnf"; } return stop_single_app_manager(); } } sub check_single_app_status { return new MHA::ManagerAdmin( app => $app, global_conf => $global_conf, conf => $conf, status_dir => $status_dir, logfile => $logfile, ping_limit => $ping_limit )->check_status(); } sub stop_single_app_manager { return new MHA::ManagerAdmin( app => $app, global_conf => $global_conf, conf => $conf, status_dir => $status_dir, logfile => $logfile, ping_limit => $ping_limit, abort => $abort, abort_timeout => $abort_timeout, )->stop_manager(); } 1; mha4mysql-manager-0.58/lib/MHA/ManagerConst.pm000066400000000000000000000066641325502577300211140ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::ManagerConst; use strict; use warnings FATAL => 'all'; use MHA::NodeConst; our $VERSION = '0.58'; our $NODE_MIN_VERSION = '0.54'; our @ALIVE_ERROR_CODES = ( 1040, # ER_CON_COUNT_ERROR 1042, # ER_BAD_HOST_ERROR 1043, # ER_HANDSHAKE_ERROR 1044, # ER_DBACCESS_DENIED_ERROR 1045, # ER_ACCESS_DENIED_ERROR 1129, # ER_HOST_IS_BLOCKED 1130, # ER_HOST_NOT_PRIVILEGED 1203, # ER_TOO_MANY_USER_CONNECTIONS 1226, # ER_USER_LIMIT_REACHED 1251, # ER_NOT_SUPPORTED_AUTH_MODE 1275, # ER_SERVER_IS_IN_SECURE_AUTH_MODE ); our $MYSQL_UNKNOWN_TID = 1094; our $MASTER_DEAD_RC = 20; our $MYSQL_DEAD_RC = 10; # Manager status our $ST_RUNNING = 0; our $ST_NOT_RUNNING = 2; our $ST_PARTIALLY_RUNNING = 3; our $ST_INITIALIZING_MONITOR = 10; our $ST_PING_FAILING = 20; our $ST_PING_FAILED = 21; our $ST_RETRYING_MONITOR = 30; our $ST_CONFIG_ERROR = 31; our $ST_TIMESTAMP_OLD = 32; our $ST_FAILOVER_RUNNING = 50; our $ST_FAILOVER_ERROR = 51; our $ST_RUNNING_S = "$ST_RUNNING:PING_OK"; our $ST_NOT_RUNNING_S = "$ST_NOT_RUNNING:NOT_RUNNING"; our $ST_PARTIALLY_RUNNING_S = "$ST_PARTIALLY_RUNNING:PARTIALLY_RUNNING"; our $ST_INITIALIZING_MONITOR_S = "$ST_INITIALIZING_MONITOR:INITIALIZING_MONITOR"; our $ST_PING_FAILING_S = "$ST_PING_FAILING:PING_FAILING"; our $ST_PING_FAILED_S = "$ST_PING_FAILED:PING_FAILED"; our $ST_RETRYING_MONITOR_S = "$ST_RETRYING_MONITOR:RETRYING_MONITOR"; our $ST_CONFIG_ERROR_S = "$ST_CONFIG_ERROR:CONFIG_ERROR"; our $ST_TIMESTAMP_OLD_S = "$ST_TIMESTAMP_OLD:TIMESTAMP_OLD"; our $ST_FAILOVER_RUNNING_S = "$ST_FAILOVER_RUNNING:FAILOVER_RUNNING"; our $ST_FAILOVER_ERROR_S = "$ST_FAILOVER_ERROR:FAILOVER_ERROR"; our $USE_SSH_OPTIONS = 0; our $SSH_OPT_ALIVE = $MHA::NodeConst::SSH_OPT_ALIVE; our $SSH_OPT_CHECK = "-o StrictHostKeyChecking=no -o PasswordAuthentication=no -o BatchMode=yes -o ConnectTimeout=VAR_CONNECT_TIMEOUT"; our $SSH_OPT_ALIVE_DEFAULT = $SSH_OPT_ALIVE; our $SSH_OPT_CHECK_DEFAULT = $SSH_OPT_CHECK; our $PING_TYPE_CONNECT = "CONNECT"; our $PING_TYPE_SELECT = "SELECT"; our $PING_TYPE_INSERT = "INSERT"; our $DEFAULT_GLOBAL_CONF = "/etc/masterha_default.cnf"; our $log_fmt = sub { my %args = @_; my $msg = $args{message}; $msg = "" unless ($msg); chomp $msg; if ( $args{level} eq "error" ) { my ( $ln, $script ) = ( caller(4) )[ 2, 1 ]; $script =~ s/.*:://; return sprintf( "[%s][%s, ln%d] %s\n", $args{level}, $script, $ln, $msg ); } return sprintf( "[%s] %s\n", $args{level}, $msg ); }; our $add_timestamp = sub { my %p = @_; sprintf "%s - %s", scalar(localtime), $p{message}; }; 1; mha4mysql-manager-0.58/lib/MHA/ManagerUtil.pm000066400000000000000000000110041325502577300207230ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::ManagerUtil; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use MHA::ManagerConst; use MHA::NodeUtil; use Log::Dispatch; use Log::Dispatch::File; use Log::Dispatch::Screen; sub init_log { my $log_output = shift; my $level = shift; $level = "info" unless ($level); my $log = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); unless ($log_output) { $log->add( Log::Dispatch::Screen->new( name => 'screen', min_level => $level, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append', ) ); } else { $log->add( Log::Dispatch::File->new( name => 'file', filename => $log_output, min_level => $level, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append', close_after_write => 1, ) ); } return $log; } sub exec_system { my $cmd = shift; my $log_output = shift; if ($log_output) { return MHA::NodeUtil::system_rc( system("$cmd >> $log_output 2>&1") ); } else { return MHA::NodeUtil::system_rc( system($cmd) ); } } sub exec_ssh_check_cmd($$$$) { my $ssh_host = shift; my $ssh_port = shift; my $ssh_cmd = shift; my $log_output = shift; my $ret; return exec_system( "ssh $MHA::ManagerConst::SSH_OPT_CHECK -p $ssh_port $ssh_host \"$ssh_cmd\"", $log_output ); } sub exec_ssh_cmd($$$$) { my $ssh_host = shift; my $ssh_port = shift; my $ssh_cmd = shift; my $log_output = shift; my $ret; return exec_system( "ssh $MHA::ManagerConst::SSH_OPT_ALIVE -p $ssh_port $ssh_host \"$ssh_cmd\"", $log_output ); } sub get_node_version { my $log = shift; my $ssh_user = shift; my $ssh_host = shift; my $ssh_ip = shift; my $ssh_port = shift; my $ssh_user_host; my $node_version; my $command = "apply_diff_relay_logs --version"; if ( $ssh_host || $ssh_ip ) { if ($ssh_ip) { $ssh_user_host = $ssh_user . '@' . $ssh_ip; } elsif ($ssh_host) { $ssh_user_host = $ssh_user . '@' . $ssh_host; } $command = "ssh $MHA::ManagerConst::SSH_OPT_ALIVE $ssh_user_host -p $ssh_port \"$command\" 2>&1"; } my $v = `$command`; chomp($v); if ( $v =~ /version (\d+\.\d+)/ ) { $node_version = $1; } else { $log->error("Got error when getting node version. Error:"); $log->error("\n$v") if ($v); } return $node_version; } sub check_node_version { my $log = shift; my $ssh_user = shift; my $ssh_host = shift; my $ssh_ip = shift; my $ssh_port = shift; my $node_version; eval { $node_version = get_node_version( $log, $ssh_user, $ssh_host, $ssh_ip, $ssh_port ); my $host = $ssh_host ? $ssh_host : $ssh_ip; croak "node version on $host not found! Is MHA Node package installed ?\n" unless ($node_version); if ( $node_version < $MHA::ManagerConst::NODE_MIN_VERSION ) { $host = "local" unless ($host); my $msg = sprintf( "Node version(%s) on %s must be equal or higher than %s.\n", $node_version, $host, $MHA::ManagerConst::NODE_MIN_VERSION ); croak $msg; } }; if ($@) { $log->error($@); die; } return $node_version; } sub check_node_version_nodie { my $log = shift; my $ssh_user = shift; my $ssh_host = shift; my $ssh_ip = shift; my $ssh_port = shift; my $rc = 1; eval { check_node_version( $log, $ssh_user, $ssh_host, $ssh_ip, $ssh_port ); $rc = 0; }; if ($@) { undef $@; } return $rc; } # should be used when it is unclear whether $log is initialized or not sub print_error { my $str = shift; my $log = shift; if ($log) { $log->error($str); } else { warn "$str\n"; } } 1; mha4mysql-manager-0.58/lib/MHA/MasterFailover.pm000066400000000000000000002222231325502577300214450ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::MasterFailover; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Carp qw(croak); use Getopt::Long qw(:config pass_through); use Log::Dispatch; use Log::Dispatch::File; use MHA::NodeUtil; use MHA::Config; use MHA::ServerManager; use MHA::FileStatus; use MHA::ManagerUtil; use MHA::ManagerConst; use MHA::HealthCheck; use MHA::SlaveUtil; use File::Basename; use Parallel::ForkManager; use Sys::Hostname; my $g_global_config_file = $MHA::ManagerConst::DEFAULT_GLOBAL_CONF; my $g_config_file; my $g_new_master_host; my $g_new_master_port = 3306; my $g_interactive = 1; my $g_ssh_reachable = 2; my $g_workdir; my $g_logfile; my $g_last_failover_minute = 480; my $g_wait_on_failover_error = 0; my $g_ignore_last_failover; my $g_skip_save_master_binlog; my $g_remove_dead_master_conf; my $g_skip_change_master; my $g_skip_disable_read_only; my $g_wait_until_gtid_in_sync = 1; my $g_ignore_binlog_server_error; my $_real_ssh_reachable; my $_saved_file_suffix; my $_start_datetime; my $_failover_complete_file; my $_failover_error_file; my %_dead_master_arg; my $_server_manager; my $_diff_binary_log; my $_diff_binary_log_basename; my $_has_saved_binlog = 0; my $_status_handler; my $_create_error_file = 0; my $log; my $mail_subject; my $mail_body; my $GEN_DIFF_OK = 15; sub exit_by_signal { $log->info("Got terminate signal during failover. Exit."); eval { MHA::NodeUtil::create_file_if($_failover_error_file) if ($_create_error_file); MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ); }; if ($@) { $log->error("Got Error: $@"); undef $@; } exit 1; } sub exec_ssh_child_cmd { my ( $ssh_user_host, $ssh_port, $ssh_cmd, $logger, $file ) = @_; my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $ssh_port, $ssh_cmd, $file ); if ( $logger && $file ) { $logger->info( "\n" . `cat $file` ); unlink $file; } return ( $high, $low ); } sub init_config() { $log = MHA::ManagerUtil::init_log($g_logfile); my ( $sc_ref, $binlog_ref ) = new MHA::Config( logger => $log, globalfile => $g_global_config_file, file => $g_config_file )->read_config(); my @servers_config = @$sc_ref; my @binlog_servers_config = @$binlog_ref; if ( !$g_logfile && !$g_interactive && $servers_config[0]->{manager_log} ) { $g_logfile = $servers_config[0]->{manager_log}; } $log = MHA::ManagerUtil::init_log( $g_logfile, $servers_config[0]->{log_level} ); $log->info("MHA::MasterFailover version $MHA::ManagerConst::VERSION."); unless ($g_workdir) { if ( $servers_config[0]->{manager_workdir} ) { $g_workdir = $servers_config[0]->{manager_workdir}; } else { $g_workdir = "/var/tmp"; } } return ( \@servers_config, \@binlog_servers_config ); } sub check_settings($) { my $servers_config_ref = shift; my @servers_config = @$servers_config_ref; my $dead_master; MHA::ManagerUtil::check_node_version($log); $_status_handler = new MHA::FileStatus( conffile => $g_config_file, dir => $g_workdir ); $_status_handler->init(); $_status_handler->set_master_host( $_dead_master_arg{hostname} ); my $appname = $_status_handler->{basename}; $_failover_complete_file = "$g_workdir/$appname.failover.complete"; $_failover_error_file = "$g_workdir/$appname.failover.error"; $_status_handler->update_status($MHA::ManagerConst::ST_FAILOVER_RUNNING_S); $_server_manager = new MHA::ServerManager( servers => \@servers_config ); $_server_manager->set_logger($log); if ($g_interactive) { $_server_manager->connect_all_and_read_server_status(); } else { $log->debug( "Skipping connecting to dead master $_dead_master_arg{hostname}."); $_server_manager->connect_all_and_read_server_status( $_dead_master_arg{hostname}, $_dead_master_arg{ip}, $_dead_master_arg{port} ); } my $m = $_server_manager->get_orig_master(); if ( !( $_dead_master_arg{hostname} eq $m->{hostname} && $_dead_master_arg{ip} eq $m->{ip} && $_dead_master_arg{port} eq $m->{port} ) ) { $log->error( sprintf( "Detected dead master %s does not match with specified dead master %s(%s:%s)!", $m->get_hostinfo(), $_dead_master_arg{hostname}, $_dead_master_arg{ip}, $_dead_master_arg{port} ) ); croak; } my @dead_servers = $_server_manager->get_dead_servers(); my @alive_servers = $_server_manager->get_alive_servers(); my @alive_slaves = $_server_manager->get_alive_slaves(); #Make sure that dead server is current master only $log->info("Dead Servers:"); $_server_manager->print_dead_servers(); if ( $#dead_servers < 0 ) { $log->error("None of server is dead. Stop failover."); croak; } my $dead_master_found = 0; foreach my $d (@dead_servers) { if ( $d->{hostname} eq $_dead_master_arg{hostname} ) { $dead_master_found = 1; $dead_master = $d; last; } } unless ($dead_master_found) { $log->error( "The master $_dead_master_arg{hostname} is not dead. Stop failover."); croak; } # quick check that the dead server is really dead # not double check when ping_type is insert, # because check_connection_fast_util can rerurn true if insert-check detects I/O failure. if ( $servers_config[0]->{ping_type} ne $MHA::ManagerConst::PING_TYPE_INSERT ) { $log->info("Checking master reachability via MySQL(double check)..."); if ( my $rc = MHA::DBHelper::check_connection_fast_util( $dead_master->{hostname}, $dead_master->{port}, $dead_master->{user}, $dead_master->{password} ) ) { $log->error( sprintf( "The master %s is reachable via MySQL (error=%s) ! Stop failover.", $dead_master->get_hostinfo(), $rc ) ); croak; } $log->info(" ok."); } $log->info("Alive Servers:"); $_server_manager->print_alive_servers(); $log->info("Alive Slaves:"); $_server_manager->print_alive_slaves(); $_server_manager->print_failed_slaves_if(); $_server_manager->print_unmanaged_slaves_if(); if ( $dead_master->{handle_raw_binlog} ) { $_saved_file_suffix = ".binlog"; } else { $_saved_file_suffix = ".sql"; } foreach my $slave (@alive_slaves) { # Master_Host is either hostname or IP address of the current master if ( $dead_master->{hostname} ne $slave->{Master_Host} && $dead_master->{ip} ne $slave->{Master_Host} && $dead_master->{hostname} ne $slave->{Master_IP} && $dead_master->{ip} ne $slave->{Master_IP} ) { $log->error( sprintf( "Slave %s does not replicate from dead master %s. Stop failover.", $slave->get_hostinfo(), $dead_master->get_hostinfo() ) ); croak; } $slave->{ssh_ok} = 2; $slave->{diff_file_readtolatest} = "$slave->{remote_workdir}/relay_from_read_to_latest_" . $slave->{hostname} . "_" . $slave->{port} . "_" . $_start_datetime . $_saved_file_suffix; } $_server_manager->validate_num_alive_servers( $dead_master, 1 ); # Checking last failover error file if ($g_ignore_last_failover) { MHA::NodeUtil::drop_file_if($_failover_error_file); MHA::NodeUtil::drop_file_if($_failover_complete_file); } if ( -f $_failover_error_file ) { my $message = "Failover error flag file $_failover_error_file " . "exists. This means the last failover failed. Check error logs " . "for detail, fix problems, remove $_failover_error_file, " . "and restart this script."; $log->error($message); croak; } if ($g_interactive) { print "Master " . $dead_master->get_hostinfo() . " is dead. Proceed? (yes/NO): "; my $ret = ; chomp($ret); die "Stopping failover." if ( lc($ret) !~ /^y/ ); } # If the last failover was done within 8 hours, we don't do failover # to avoid ping-pong if ( -f $_failover_complete_file ) { my $lastts = ( stat($_failover_complete_file) )[9]; my $current_time = time(); if ( $current_time - $lastts < $g_last_failover_minute * 60 ) { my ( $sec, $min, $hh, $dd, $mm, $yy, $week, $yday, $opt ) = localtime($lastts); my $t = sprintf( "%04d/%02d/%02d %02d:%02d:%02d", $yy + 1900, $mm + 1, $dd, $hh, $min, $sec ); my $msg = "Last failover was done at $t." . " Current time is too early to do failover again. If you want to " . "do failover, manually remove $_failover_complete_file " . "and run this script again."; $log->error($msg); croak; } else { MHA::NodeUtil::drop_file_if($_failover_complete_file); } } $_server_manager->get_failover_advisory_locks(); $_server_manager->start_sql_threads_if(); return $dead_master; } sub force_shutdown_internal($) { my $dead_master = shift; $log->info( "Forcing shutdown so that applications never connect to the current master.." ); if ( $dead_master->{master_ip_failover_script} ) { my $command = "$dead_master->{master_ip_failover_script} --orig_master_host=$dead_master->{hostname} --orig_master_ip=$dead_master->{ip} --orig_master_port=$dead_master->{port}"; if ( $_real_ssh_reachable == 1 ) { $command .= " --command=stopssh" . " --ssh_user=$dead_master->{ssh_user} "; } else { $command .= " --command=stop"; } $command .= $dead_master->get_ssh_args_if( 1, "orig", $_real_ssh_reachable ); $log->info("Executing master IP deactivation script:"); $log->info(" $command"); my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $g_logfile ); if ( $high == 0 && $low == 0 ) { $log->info(" done."); $mail_body .= "Invalidated master IP address on " . $dead_master->get_hostinfo() . "\n"; } else { my $message = "Failed to deactivate master IP with return code $high:$low"; $log->error($message); $mail_body .= $message . "\n"; if ( $high == 10 ) { $log->warning("Proceeding."); } else { croak; } } } else { $log->warning( "master_ip_failover_script is not set. Skipping invalidating dead master IP address." ); } # force master shutdown if ( $dead_master->{shutdown_script} ) { my $command = "$dead_master->{shutdown_script}"; if ( $_real_ssh_reachable == 1 ) { $command .= " --command=stopssh" . " --ssh_user=$dead_master->{ssh_user} "; } else { $command .= " --command=stop"; } $command .= " --host=$dead_master->{hostname} --ip=$dead_master->{ip} --port=$dead_master->{port} "; $command .= " --pid_file=$dead_master->{master_pid_file}" if ( $dead_master->{master_pid_file} ); $command .= $dead_master->get_ssh_args_if( 1, "shutdown", $_real_ssh_reachable ); $log->info("Executing SHUTDOWN script:"); $log->info(" $command"); my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $g_logfile ); if ( $high == 0 && $low == 0 ) { $log->info(" Power off done."); $mail_body .= "Power off $dead_master->{hostname}.\n"; $_real_ssh_reachable = 0; } else { if ( $high == 10 ) { $log->info(" SSH reachable. Shutting down mysqld done."); $mail_body .= "SSH reachable on $dead_master->{hostname}. Shutting down mysqld done.\n"; $_real_ssh_reachable = 1; } else { my $message = "Failed to execute shutdown_script with return code $high:$low"; $log->error($message); $mail_body .= $message . "\n"; croak; } } } else { $log->warning( "shutdown_script is not set. Skipping explicit shutting down of the dead master." ); } return 0; } sub force_shutdown($) { my $dead_master = shift; my $appname = $_status_handler->{basename}; my @alive_slaves = $_server_manager->get_alive_slaves(); $mail_subject = $appname . ": MySQL Master failover " . $dead_master->get_hostinfo(); $mail_body = "Master " . $dead_master->get_hostinfo() . " is down!\n\n"; $mail_body .= "Check MHA Manager logs at " . hostname(); $mail_body .= ":$g_logfile" if ($g_logfile); $mail_body .= " for details.\n\n"; if ($g_interactive) { $mail_body .= "Started manual(interactive) failover.\n"; } else { $mail_body .= "Started automated(non-interactive) failover.\n"; } # If any error happens after here, a special error file is created so that # it won't automatically repeat the same error. $_create_error_file = 1; my $slave_io_stopper = new Parallel::ForkManager( $#alive_slaves + 1 ); my $stop_io_failed = 0; $slave_io_stopper->run_on_start( sub { my ( $pid, $target ) = @_; } ); $slave_io_stopper->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; return if ( $target->{ignore_fail} ); $stop_io_failed = 1 if ($exit_code); } ); foreach my $target (@alive_slaves) { $slave_io_stopper->start($target) and next; eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; my $rc = $target->stop_io_thread(); $slave_io_stopper->finish($rc); }; if ($@) { $log->error($@); undef $@; $slave_io_stopper->finish(1); } $slave_io_stopper->finish(0); } $_real_ssh_reachable = $g_ssh_reachable; # SSH reachability is unknown. Verify here. if ( $_real_ssh_reachable >= 2 ) { if ( MHA::HealthCheck::ssh_check_simple( $dead_master->{ssh_user}, $dead_master->{ssh_host}, $dead_master->{ssh_ip}, $dead_master->{ssh_port}, $dead_master->{logger}, $dead_master->{ssh_connection_timeout} ) ) { $_real_ssh_reachable = 0; } else { # additional check if ( MHA::ManagerUtil::get_node_version( $dead_master->{logger}, $dead_master->{ssh_user}, $dead_master->{ssh_host}, $dead_master->{ssh_ip}, $dead_master->{ssh_port} ) ) { $_real_ssh_reachable = 1; } else { $log->warning( "Failed to get MHA Node version from dead master. Guessing that SSH is NOT reachable." ); $_real_ssh_reachable = 0; } } } force_shutdown_internal($dead_master); $slave_io_stopper->wait_all_children; if ($stop_io_failed) { $log->error("Stopping IO thread failed! Check slave status!"); $mail_body .= "Stopping IO thread failed.\n"; croak; } } sub check_set_latest_slaves { $_server_manager->read_slave_status(); $_server_manager->identify_latest_slaves(); $log->info( "Latest slaves (Slaves that received relay log files to the latest):"); $_server_manager->print_latest_slaves(); $_server_manager->identify_oldest_slaves(); $log->info("Oldest slaves:"); $_server_manager->print_oldest_slaves(); } sub save_from_binlog_server { my $relay_master_log_file = shift; my $exec_master_log_pos = shift; my $binlog_server_ref = shift; my @binlog_servers = @$binlog_server_ref; my $max_saved_binlog_size = 0; my $failed_servers = 0; my $pm = new Parallel::ForkManager( $#binlog_servers + 1 ); $pm->run_on_start( sub { my ( $pid, $target ) = @_; $log->info( sprintf( "-- Saving binlog from host %s started, pid: %d", $target->{hostname}, $pid ) ); } ); $pm->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; $log->info(); $log->info("Log messages from $target->{hostname} ..."); my $saved_binlog = "$g_workdir/saved_binlog_$target->{hostname}_$target->{id}_$_start_datetime.binlog"; my $local_file = "$g_workdir/saved_binlog_$target->{hostname}_$target->{id}_$_start_datetime.log"; if ( -f $local_file ) { $log->info( "\n" . `cat $local_file` ); unlink $local_file; } $log->info("End of log messages from $target->{hostname}."); if ( $exit_code == 0 ) { if ( -f $saved_binlog ) { my $saved_binlog_size = -s $saved_binlog; $log->info( "Saved mysqlbinlog size from $target->{hostname} is $saved_binlog_size bytes." ); if ( $saved_binlog_size > $max_saved_binlog_size ) { $_diff_binary_log = $saved_binlog; $max_saved_binlog_size = $saved_binlog_size; } } } elsif ( $exit_code == 2 ) { $failed_servers++; $log->warning("SSH is not reachable on $target->{hostname}. Skipping"); } elsif ( $exit_code == 10 ) { $log->info("No binlog events found from $target->{hostname}. Skipping"); } else { $failed_servers++; $log->warning("Got error from $target->{hostname}."); } } ); foreach my $target (@binlog_servers) { my $pid = $pm->start($target) and next; my $pplog; eval { $pm->finish(2) unless ( $target->{ssh_reachable} ); $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; my $saved_binlog = "$g_workdir/saved_binlog_$target->{hostname}_$target->{id}_$_start_datetime.binlog"; my $saved_binlog_remote = "$target->{remote_workdir}/saved_binlog_$target->{id}_$_start_datetime.binlog"; my $local_file = "$g_workdir/saved_binlog_$target->{hostname}_$target->{id}_$_start_datetime.log"; if ( -f $local_file ) { unlink $local_file; } $pplog = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); $pplog->add( Log::Dispatch::File->new( name => 'file', filename => $local_file, min_level => $target->{log_level}, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append' ) ); $pplog->info( "Fetching binary logs from binlog server $target->{hostname}.."); my $command = "save_binary_logs --command=save --start_file=$relay_master_log_file --start_pos=$exec_master_log_pos --output_file=$saved_binlog_remote --handle_raw_binlog=0 --skip_filter=1 --disable_log_bin=0 --manager_version=$MHA::ManagerConst::VERSION"; if ( $target->{client_bindir} ) { $command .= " --client_bindir=$target->{client_bindir}"; } if ( $target->{client_libdir} ) { $command .= " --client_libdir=$target->{client_libdir}"; } my $oldest_version = $_server_manager->get_oldest_version(); $command .= " --oldest_version=$oldest_version "; if ( $target->{log_level} eq "debug" ) { $command .= " --debug "; } $command .= " --binlog_dir=$target->{master_binlog_dir} "; $pplog->info("Executing binlog save command: $command"); my $ssh_user_host = $target->{ssh_user} . '@' . $target->{ssh_ip}; my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $target->{ssh_port}, $command, $local_file ); if ( $high == 0 && $low == 0 ) { if ( MHA::NodeUtil::file_copy( 0, $saved_binlog, $saved_binlog_remote, $target->{ssh_user}, $target->{ssh_ip}, $local_file, $target->{ssh_port} ) ) { $pplog->error( "scp from $ssh_user_host:$saved_binlog_remote to local:$saved_binlog failed!" ); croak; } else { $pplog->info( "scp from $ssh_user_host:$saved_binlog_remote to local:$saved_binlog succeeded." ); $pm->finish(0); } } elsif ( $high == 10 && $low == 0 ) { $pplog->info( "Additional events were not found from the binlog server. No need to save." ); $pm->finish(10); } else { $pplog->error( "Failed to save binary log events from the binlog server. Maybe disks on binary logs are not accessible or binary log itself is corrupt?" ); } }; if ($@) { $pplog->error($@) if ($pplog); undef $@; } $pm->finish(1); } $pm->wait_all_children; if (!$g_ignore_binlog_server_error && $#binlog_servers >= 0 && $#binlog_servers + 1 <= $failed_servers ) { $log->error("All binlog servers failed!"); croak; } if ($_diff_binary_log) { return 1; } else { return 0; } } sub save_master_binlog_internal { my $master_log_file = shift; my $read_master_log_pos = shift; my $dead_master = shift; $log->info("Fetching dead master's binary logs.."); $_diff_binary_log_basename = "saved_master_binlog_from_" . $dead_master->{hostname} . "_" . $dead_master->{port} . "_" . $_start_datetime . $_saved_file_suffix; $_diff_binary_log = "$g_workdir/$_diff_binary_log_basename"; my $_diff_binary_log_remote = "$dead_master->{remote_workdir}/$_diff_binary_log_basename"; if ( -f $_diff_binary_log ) { unlink($_diff_binary_log); } my $command = "save_binary_logs --command=save --start_file=$master_log_file --start_pos=$read_master_log_pos --binlog_dir=$dead_master->{master_binlog_dir} --output_file=$_diff_binary_log_remote --handle_raw_binlog=$dead_master->{handle_raw_binlog} --disable_log_bin=$dead_master->{disable_log_bin} --manager_version=$MHA::ManagerConst::VERSION"; if ( $dead_master->{client_bindir} ) { $command .= " --client_bindir=$dead_master->{client_bindir}"; } if ( $dead_master->{client_libdir} ) { $command .= " --client_libdir=$dead_master->{client_libdir}"; } unless ( $dead_master->{handle_raw_binlog} ) { my $oldest_version = $_server_manager->get_oldest_version(); $command .= " --oldest_version=$oldest_version "; } if ( $dead_master->{log_level} eq "debug" ) { $command .= " --debug "; } my $ssh_user_host = $dead_master->{ssh_user} . '@' . $dead_master->{ssh_ip}; $log->info( sprintf( "Executing command on the dead master %s: %s", $dead_master->get_hostinfo(), $command ) ); my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $dead_master->{ssh_port}, $command, $g_logfile ); if ( $high == 0 && $low == 0 ) { if ( MHA::NodeUtil::file_copy( 0, $_diff_binary_log, $_diff_binary_log_remote, $dead_master->{ssh_user}, $dead_master->{ssh_ip}, $g_logfile, $dead_master->{ssh_port} ) ) { $log->error( "scp from $ssh_user_host:$_diff_binary_log_remote to local:$_diff_binary_log failed!" ); croak; } else { $log->info( "scp from $ssh_user_host:$_diff_binary_log_remote to local:$_diff_binary_log succeeded." ); $_has_saved_binlog = 1; } } elsif ( $high == 10 && $low == 0 ) { $log->info( "Additional events were not found from the orig master. No need to save." ); } else { $log->error( "Failed to save binary log events from the orig master. Maybe disks on binary logs are not accessible or binary log itself is corrupt?" ); } if ($_has_saved_binlog) { my @alive_slaves = $_server_manager->get_alive_slaves(); foreach my $slave (@alive_slaves) { $slave->check_set_ssh_status( $log, 1 ); } } } sub save_master_binlog { my $dead_master = shift; if ( $_real_ssh_reachable && !$g_skip_save_master_binlog ) { MHA::ManagerUtil::check_node_version( $log, $dead_master->{ssh_user}, $dead_master->{ssh_host}, $dead_master->{ssh_ip}, $dead_master->{ssh_port} ); my $latest_file = ( $_server_manager->get_latest_slaves() )[0]->{Master_Log_File}; my $latest_pos = ( $_server_manager->get_latest_slaves() )[0]->{Read_Master_Log_Pos}; save_master_binlog_internal( $latest_file, $latest_pos, $dead_master, ); } else { if ($g_skip_save_master_binlog) { $log->info("Skipping trying to save dead master's binary log."); } elsif ( !$_real_ssh_reachable ) { $log->warning( "Dead Master is not SSH reachable. Could not save it's binlogs. Transactions that were not sent to the latest slave (Read_Master_Log_Pos to the tail of the dead master's binlog) were lost." ); } } } sub find_slave_with_all_relay_logs { my $oldest_master_log_file = shift; my $oldest_master_log_pos = shift; my $skip_ssh_check = shift; my @latest = $_server_manager->get_latest_slaves(); foreach my $latest_slave (@latest) { if ( !$latest_slave->{dead} && !$skip_ssh_check ) { # Need to check ssh connectivity if it is not confirmed $latest_slave->check_set_ssh_status( $log, 0 ) if ( $latest_slave->{ssh_ok} >= 2 ); next if ( $latest_slave->{ssh_ok} == 0 ); } my $ssh_user_host = $latest_slave->{ssh_user} . '@' . $latest_slave->{ssh_ip}; my $command = "apply_diff_relay_logs --command=find --latest_mlf=$latest_slave->{Master_Log_File} --latest_rmlp=$latest_slave->{Read_Master_Log_Pos} --target_mlf=$oldest_master_log_file --target_rmlp=$oldest_master_log_pos --server_id=$latest_slave->{server_id} --workdir=$latest_slave->{remote_workdir} --timestamp=$_start_datetime --manager_version=$MHA::ManagerConst::VERSION"; if ( $latest_slave->{client_bindir} ) { $command .= " --client_bindir=$latest_slave->{client_bindir}"; } if ( $latest_slave->{client_libdir} ) { $command .= " --client_libdir=$latest_slave->{client_libdir}"; } if ( $latest_slave->{relay_log_info_type} eq "TABLE" ) { $command .= " --relay_dir=$latest_slave->{relay_dir} --current_relay_log=$latest_slave->{current_relay_log} "; } else { $command .= " --relay_log_info=$latest_slave->{relay_log_info} "; $command .= " --relay_dir=$latest_slave->{datadir} "; } if ( $latest_slave->{log_level} eq "debug" ) { $command .= " --debug "; } $log->info( "Checking whether $latest_slave->{hostname} has relay logs from the oldest position.." ); if ($MHA::ManagerConst::USE_SSH_OPTIONS) { $command .= " --ssh_options='$MHA::NodeConst::SSH_OPT_ALIVE' "; } $log->info("Executing command: $command :"); my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $latest_slave->{ssh_port}, $command, $g_logfile ); if ( $high eq '0' && $low eq '0' ) { $log->info("OK. $latest_slave->{hostname} has all relay logs."); return $latest_slave; } else { $log->warning( "$latest_slave->{hostname} doesn't have all relay logs. Maybe some logs were purged." ); } } } sub find_latest_base_slave_internal { my $oldest_slave = ( $_server_manager->get_oldest_slaves() )[0]; my @latest_slaves = $_server_manager->get_latest_slaves(); my $oldest_mlf = $oldest_slave->{Master_Log_File}; my $oldest_mlp = $oldest_slave->{Read_Master_Log_Pos}; my $latest_mlf = $latest_slaves[0]->{Master_Log_File}; my $latest_mlp = $latest_slaves[0]->{Read_Master_Log_Pos}; if ( $_server_manager->pos_cmp( $oldest_mlf, $oldest_mlp, $latest_mlf, $latest_mlp ) >= 0 ) { $log->info( "All slaves received relay logs to the same position. No need to resync each other." ); return $latest_slaves[0]; } else { # We pick relay logs here. This should not reconfigure slave settings until other slaves connect to new master my $target = find_slave_with_all_relay_logs( $oldest_slave->{Master_Log_File}, $oldest_slave->{Read_Master_Log_Pos} ); unless ($target) { $log->warning( "None of latest servers have enough relay logs from oldest position. We can't recover oldest slaves." ); my @oldest_slaves = $_server_manager->get_oldest_slaves(); foreach (@oldest_slaves) { $_->{lack_relay_log} = 1; $_->{no_master} = 1; } my ( $oldest_limit_mlf, $oldest_limit_mlp ) = $_server_manager->get_oldest_limit_pos(); unless ($oldest_limit_mlf) { $log->warning( "All slave servers set ignore_fail parameters. Continuing failover."); $target = $latest_slaves[0]; } elsif ( $_server_manager->pos_cmp( $oldest_slave->{Master_Log_File}, $oldest_slave->{Read_Master_Log_Pos}, $oldest_limit_mlf, $oldest_limit_mlp ) >= 0 ) { # None of the slave sets ignore_fail parameters. Can't continue failover return; } else { $log->warning( sprintf( "The oldest master position from non-ignore_fail slaves is %s:%d. Checking whether latest slave's relay logs from this position.", $oldest_limit_mlf, $oldest_limit_mlp ) ); if ( $_server_manager->pos_cmp( $oldest_limit_mlf, $oldest_limit_mlp, $latest_mlf, $latest_mlp ) >= 0 ) { $log->info( "The oldest master position from non-ignore_fail slaves is equal to the latest slave. Can continue failover." ); return $latest_slaves[0]; } $target = find_slave_with_all_relay_logs( $oldest_limit_mlf, $oldest_limit_mlp, 1 ); $_server_manager->set_no_master_if_older( $oldest_limit_mlf, $oldest_limit_mlp ); } } return $target; } } sub find_latest_base_slave($) { my $dead_master = shift; $log->info( "Finding the latest slave that has all relay logs for recovering other slaves.." ); my $latest_base_slave = find_latest_base_slave_internal(); unless ($latest_base_slave) { my $msg = "None of the latest slaves has enough relay logs for recovery."; $log->error($msg); $mail_body .= $msg . "\n"; croak; } $mail_body .= "The latest slave " . $latest_base_slave->get_hostinfo() . " has all relay logs for recovery.\n"; reconf_alive_servers($dead_master); return $latest_base_slave; } sub select_new_master($$) { my $dead_master = shift; my $latest_base_slave = shift; my $new_master = $_server_manager->select_new_master( $g_new_master_host, $g_new_master_port, $latest_base_slave->{check_repl_delay} ); unless ($new_master) { my $msg = "None of existing slaves matches as a new master. Maybe preferred node is misconfigured or all slaves are too far behind."; $log->error($msg); $mail_body .= $msg . "\n"; croak; } $log->info( "New master is " . $new_master->get_hostinfo() ); $mail_body .= "Selected " . $new_master->get_hostinfo() . " as a new master.\n"; $log->info("Starting master failover.."); $_server_manager->print_servers_migration_ascii( $dead_master, $new_master ); if ($g_interactive) { $new_master = $_server_manager->manually_decide_new_master( $dead_master, $new_master ); $log->info( "New master decided manually is " . $new_master->get_hostinfo() ); } return $new_master; } sub send_binlog { my ( $target, $logger ) = @_; $logger = $log unless ($logger); if ($_has_saved_binlog) { $logger->info("Sending binlog.."); my $_diff_binary_log_remote = "$target->{remote_workdir}/$_diff_binary_log_basename"; if ( MHA::NodeUtil::file_copy( 1, $_diff_binary_log, $_diff_binary_log_remote, $target->{ssh_user}, $target->{ssh_ip}, $g_logfile, $target->{ssh_port} ) ) { $logger->error( "scp from local:$_diff_binary_log to $target->{ssh_user}" . '@' . $target->{hostname} . "$_diff_binary_log_remote failed." ); return 1; } else { $logger->info( "scp from local:$_diff_binary_log to $target->{ssh_user}" . '@' . $target->{hostname} . ":$_diff_binary_log_remote succeeded." ); } } return 0; } sub generate_diff_from_readpos { my ( $target, $latest_slave, $logger ) = @_; $logger = $log unless ($logger); $logger->info( "Server $target->{hostname} received relay logs up to: $target->{Master_Log_File}:$target->{Read_Master_Log_Pos}" ); $logger->info( "Need to get diffs from the latest slave($latest_slave->{hostname}) up to: $latest_slave->{Master_Log_File}:$latest_slave->{Read_Master_Log_Pos} (using the latest slave's relay logs)" ); my $ssh_user_host = $latest_slave->{ssh_user} . '@' . $latest_slave->{ssh_ip}; my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $latest_slave->{ssh_port}, "exit 0", $g_logfile ); if ( $high ne '0' || $low ne '0' ) { $logger->error("SSH authentication test failed. user=$ssh_user_host"); return ( $high, $low ); } $logger->info( "Connecting to the latest slave host $latest_slave->{hostname}, generating diff relay log files.." ); my $command = "apply_diff_relay_logs --command=generate_and_send --scp_user=$target->{ssh_user} --scp_host=$target->{ssh_ip} --latest_mlf=$latest_slave->{Master_Log_File} --latest_rmlp=$latest_slave->{Read_Master_Log_Pos} --target_mlf=$target->{Master_Log_File} --target_rmlp=$target->{Read_Master_Log_Pos} --server_id=$latest_slave->{server_id} --diff_file_readtolatest=$target->{diff_file_readtolatest} --workdir=$latest_slave->{remote_workdir} --timestamp=$_start_datetime --handle_raw_binlog=$target->{handle_raw_binlog} --disable_log_bin=$target->{disable_log_bin} --manager_version=$MHA::ManagerConst::VERSION"; if ( $latest_slave->{client_bindir} ) { $command .= " --client_bindir=$latest_slave->{client_bindir}"; } if ( $latest_slave->{client_libdir} ) { $command .= " --client_libdir=$latest_slave->{client_libdir}"; } if ( $target->{ssh_port} ne 22 ) { $command .= " --scp_port=$target->{ssh_port}"; } if ( $latest_slave->{relay_log_info_type} eq "TABLE" ) { $command .= " --relay_dir=$latest_slave->{relay_dir} --current_relay_log=$latest_slave->{current_relay_log} "; } else { $command .= " --relay_log_info=$latest_slave->{relay_log_info} "; $command .= " --relay_dir=$latest_slave->{datadir} "; } unless ( $target->{handle_raw_binlog} ) { $command .= " --target_version=$target->{mysql_version} "; } if ( $target->{log_level} eq "debug" ) { $command .= " --debug "; } if ($MHA::ManagerConst::USE_SSH_OPTIONS) { $command .= " --ssh_options='$MHA::NodeConst::SSH_OPT_ALIVE' "; } $logger->info("Executing command: $command"); return exec_ssh_child_cmd( $ssh_user_host, $latest_slave->{ssh_port}, $command, $logger, "$g_workdir/$latest_slave->{hostname}_$latest_slave->{port}.work" ); } # 0: no need to generate diff # 15: generating diff succeeded # 1: fail sub recover_relay_logs { my ( $target, $latest_slave, $logger ) = @_; $logger = $log unless ($logger); if ( $target->{latest} eq '0' ) { my ( $high, $low ) = generate_diff_from_readpos( $target, $latest_slave, $logger ); if ( $high ne '0' || $low ne '0' ) { $logger->error( " Generating diff files failed with return code $high:$low."); return 1; } $logger->info(" Generating diff files succeeded."); return $GEN_DIFF_OK; } else { $logger->info( " This server has all relay logs. No need to generate diff files from the latest slave." ); return 0; } } sub recover_all_slaves_relay_logs { my $new_master = shift; my $latest_base_slave = shift; my @alive_slaves = $_server_manager->get_alive_slaves(); $log->info( "* Phase 4.1: Starting Parallel Slave Diff Log Generation Phase..\n"); $log->info(); my $pm = new Parallel::ForkManager( $#alive_slaves + 1 ); my $diff_log_fail = 0; $pm->run_on_start( sub { my ( $pid, $target ) = @_; $log->info( sprintf( "-- Slave diff file generation on host %s started, pid: %d. Check tmp log $g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log if it takes time..", $target->get_hostinfo(), $pid ) ); } ); $pm->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; $log->info(); $log->info("Log messages from $target->{hostname} ..."); my $local_file = "$g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log"; $log->info( "\n" . `cat $local_file` ) if ( -f $local_file ); $log->info("End of log messages from $target->{hostname}."); unlink $local_file if ( -f $local_file ); if ( $exit_code == 0 ) { $target->{gen_diff_ok} = 1; $log->info( sprintf( "-- %s has the latest relay log events.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": This host has the latest relay log events.\n"; } elsif ( $exit_code == $GEN_DIFF_OK ) { $target->{gen_diff_ok} = 1; $log->info( sprintf( "-- Slave diff log generation on host %s succeeded.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": Generating differential relay logs up to " . $latest_base_slave->get_hostinfo() . "succeeded.\n"; } else { $diff_log_fail = 1; $log->info( sprintf( "-- Slave diff log generation on host %s failed, exit code %d", $target->get_hostinfo(), $exit_code ) ); $mail_body .= $target->get_hostinfo() . ": Generating differential relay logs up to " . $latest_base_slave->get_hostinfo() . " failed.\n"; } } ); foreach my $target (@alive_slaves) { # master was already recovered next if ( $target->{id} eq $new_master->{id} ); my $pid = $pm->start($target) and next; my $pplog; eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; $pm->finish(2) if ( $target->{lack_relay_log} ); my $local_file = "$g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log"; unlink $local_file; $pplog = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); $pplog->add( Log::Dispatch::File->new( name => 'file', filename => $local_file, min_level => $target->{log_level}, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append' ) ); $target->current_slave_position(); my $rc = recover_relay_logs( $target, $latest_base_slave, $pplog ); $pm->finish(0) if ( $rc == 0 ); $pm->finish($GEN_DIFF_OK) if ( $rc == $GEN_DIFF_OK ); $pm->finish(1); }; if ($@) { $pplog->error($@) if ($pplog); undef $@; $pm->finish(1); } } $pm->wait_all_children; return 1 if ($diff_log_fail); return 0; } sub gen_diff_from_exec_to_read { my ( $target, $logger ) = @_; $logger = $log unless ($logger); my $ret; if ( ( $target->{Master_Log_File} eq $target->{Relay_Master_Log_File} ) && ( $target->{Read_Master_Log_Pos} == $target->{Exec_Master_Log_Pos} ) ) { $logger->info( sprintf( "This slave(%s)'s Exec_Master_Log_Pos equals to Read_Master_Log_Pos(%s:%d). No need to recover from Exec_Master_Log_Pos.", $target->{hostname}, $target->{Master_Log_File}, $target->{Read_Master_Log_Pos} ) ); return 0; } else { $logger->info( sprintf( "This slave(%s)'s Exec_Master_Log_Pos(%s:%d) does not equal to Read_Master_Log_Pos(%s:%d). It is likely that relay log was cut during transaction. Need to recover from Exec_Master_Log_Pos.", $target->{hostname}, $target->{Relay_Master_Log_File}, $target->{Exec_Master_Log_Pos}, $target->{Master_Log_File}, $target->{Read_Master_Log_Pos} ) ); $logger->info( "Saving local relay logs from exec pos to read pos on $target->{hostname}: from $target->{Relay_Log_File}:$target->{Relay_Log_Pos} to the end of the relay log.." ); $target->{relay_from_exectoread} = "$target->{remote_workdir}/relay_from_exec_to_read_" . $target->{hostname} . "_" . $target->{port} . "_" . $_start_datetime . $_saved_file_suffix; my $command = "save_binary_logs --command=save --start_file=$target->{Relay_Log_File} --start_pos=$target->{Relay_Log_Pos} --output_file=$target->{relay_from_exectoread} --handle_raw_binlog=$target->{handle_raw_binlog} --disable_log_bin=$target->{disable_log_bin} --manager_version=$MHA::ManagerConst::VERSION"; if ( $target->{client_bindir} ) { $command .= " --client_bindir=$target->{client_bindir}"; } if ( $target->{client_libdir} ) { $command .= " --client_libdir=$target->{client_libdir}"; } if ( $target->{relay_log_info_type} eq "TABLE" ) { $command .= " --binlog_dir=$target->{relay_dir} "; } else { $command .= " --relay_log_info=$target->{relay_log_info} "; $command .= " --binlog_dir=$target->{datadir} "; } unless ( $target->{handle_raw_binlog} ) { $command .= " --oldest_version=$target->{mysql_version} "; } if ( $target->{log_level} eq "debug" ) { $command .= " --debug "; } $logger->info("Executing command : $command"); my $ssh_user_host = $target->{ssh_user} . '@' . $target->{ssh_ip}; $target->check_set_ssh_status( $logger, 1 ) if ( $target->{ssh_ok} >= 2 ); if ( $target->{ssh_ok} == 0 ) { $logger->error("Failed to connect via SSH!"); return 1; } my ( $high, $low ) = exec_ssh_child_cmd( $ssh_user_host, $target->{ssh_port}, $command, $logger, "$g_workdir/$target->{hostname}_$target->{port}.work" ); if ( $high eq '0' && $low eq '0' ) { return 0; } else { $logger->error("Saving relay logs failed!"); return 1; } } return 0; } sub apply_diff { my ( $target, $logger ) = @_; $logger = $log unless ($logger); my $ret = 0; $logger->info("Waiting until all relay logs are applied."); $ret = $target->wait_until_relay_log_applied(); if ($ret) { $logger->error("Applying existing relay logs failed!"); return $ret; } $logger->info(" done."); $target->stop_sql_thread($logger); $logger->info("Getting slave status.."); my %status = $target->check_slave_status(); if ( $status{Status} eq '0' ) { $target->{Relay_Master_Log_File} = $status{Relay_Master_Log_File}; $target->{Exec_Master_Log_Pos} = $status{Exec_Master_Log_Pos}; $target->{Relay_Log_File} = $status{Relay_Log_File}; $target->{Relay_Log_Pos} = $status{Relay_Log_Pos}; } else { $logger->error("Getting slave status failed"); return $status{Status}; } # generate from exec pos to my latest pos $ret = gen_diff_from_exec_to_read( $target, $logger ); return ( $ret, 0 ) if ($ret); # if exec pos != read pos (relay logs cut in transactions) my $exec_diff = $target->{relay_from_exectoread}; # if read pos != latest (io thread) my $read_diff = $target->{diff_file_readtolatest}; # if binlogs are rescued from master my $binlog_diff; if ($_diff_binary_log_basename) { $binlog_diff = "$target->{remote_workdir}/$_diff_binary_log_basename"; } my @diffs; push @diffs, $exec_diff if ($exec_diff); push @diffs, $read_diff if ( $read_diff && !$target->{latest} ); push @diffs, $binlog_diff if ( $binlog_diff && $_has_saved_binlog ); my $diff_files = join( ",", @diffs ); $target->get_and_set_high_max_allowed_packet($logger); my $ssh_user_host = $target->{ssh_user} . '@' . $target->{ssh_ip}; $logger->info( "Connecting to the target slave host $target->{hostname}, running recover script.." ); my $command = "apply_diff_relay_logs --command=apply --slave_user=$target->{escaped_user} --slave_host=$target->{hostname} --slave_ip=$target->{ip} --slave_port=$target->{port} --apply_files=$diff_files --workdir=$target->{remote_workdir} --target_version=$target->{mysql_version} --timestamp=$_start_datetime --handle_raw_binlog=$target->{handle_raw_binlog} --disable_log_bin=$target->{disable_log_bin} --manager_version=$MHA::ManagerConst::VERSION"; if ( $target->{client_bindir} ) { $command .= " --client_bindir=$target->{client_bindir}"; } if ( $target->{client_libdir} ) { $command .= " --client_libdir=$target->{client_libdir}"; } if ( $target->{log_level} eq "debug" ) { $command .= " --debug "; } if ($MHA::ManagerConst::USE_SSH_OPTIONS) { $command .= " --ssh_options='$MHA::NodeConst::SSH_OPT_ALIVE' "; } $logger->info("Executing command: $command --slave_pass=xxx"); if ( $target->{escaped_password} ne "" ) { $command .= " --slave_pass=$target->{escaped_password} "; } $target->check_set_ssh_status( $logger, 1 ) if ( $target->{ssh_ok} >= 2 ); if ( $target->{ssh_ok} == 0 ) { $logger->error("Failed to connect via SSH!"); return ( 1, 0 ); } my ( $high, $low ) = exec_ssh_child_cmd( $ssh_user_host, $target->{ssh_port}, $command, $logger, "$g_workdir/$target->{hostname}_$target->{port}.work" ); $target->set_default_max_allowed_packet($logger); return ( $high, $low ); } # apply diffs to master and get master status # We do not reset slave here sub recover_slave { my ( $target, $logger ) = @_; $logger = $log unless ($logger); $logger->info( sprintf( "Starting recovery on %s(%s:%d)..", $target->{hostname}, $target->{ip}, $target->{port} ) ); if ( $target->{latest} eq '0' || $_has_saved_binlog ) { $logger->info(" Generating diffs succeeded."); my ( $high, $low ) = apply_diff( $target, $logger ); if ( $high ne '0' || $low ne '0' ) { $logger->error(" Applying diffs failed with return code $high:$low."); return -1; } } else { $logger->info( " This server has all relay logs. Waiting all logs to be applied.. "); my $ret = $target->wait_until_relay_log_applied($logger); if ($ret) { $logger->error(" Failed with return code $ret"); return -1; } $logger->info(" done."); $target->stop_sql_thread($logger); } $logger->info(" All relay logs were successfully applied."); return 0; } sub apply_binlog_to_master($) { my $target = shift; my $err_file = "$g_workdir/mysql_from_binlog.err"; my $command = "cat $_diff_binary_log | mysql --binary-mode --user=$target->{mysql_escaped_user} --password=$target->{mysql_escaped_password} --host=$target->{ip} --port=$target->{port} -vvv --unbuffered > $err_file 2>&1"; $log->info("Checking if super_read_only is defined and turned on.."); my ($super_read_only_enabled, $dbh) = MHA::SlaveUtil::check_if_super_read_only($target->{hostname}, $target->{ip}, $target->{port}, $target->{user}, $target->{password}); if ($super_read_only_enabled) { MHA::SlaveUtil::disable_super_read_only($dbh); } else { $log->info(" not present or turned off, ignoring.\n"); } $log->info("Applying differential binlog $_diff_binary_log .."); if ( my $rc = system($command) ) { my ( $high, $low ) = MHA::NodeUtil::system_rc($rc); $log->error("FATAL: applying log files failed with rc $high:$low!"); $log->error( sprintf( "Error logs from %s:%s (the last 200 lines)..", $target->{hostname}, $err_file ) ); $log->error(`tail -200 $err_file`); if ($super_read_only_enabled) { $log->info("Enabling super_read_only again after failure\n"); MHA::SlaveUtil::enable_super_read_only($dbh); } croak; } else { $log->info("Differential log apply from binlog server succeeded."); } if ($super_read_only_enabled) { $log->info("Enabling super_read_only again after applying\n"); MHA::SlaveUtil::enable_super_read_only($dbh); } return 0; } sub recover_master_gtid_internal($$$) { my $target = shift; my $latest_slave = shift; my $binlog_server_ref = shift; my $relay_master_log_file; my $exec_master_log_pos; $log->info(); $log->info("* Phase 3.3: New Master Recovery Phase..\n"); $log->info(); $log->info(" Waiting all logs to be applied.. "); my $ret = $target->wait_until_relay_log_applied($log); if ($ret) { $log->error(" Failed with return code $ret"); return -1; } $log->info(" done."); $target->stop_slave($log); if ( $target->{id} ne $latest_slave->{id} ) { $log->info( sprintf( " Replicating from the latest slave %s and waiting to apply..", $latest_slave->get_hostinfo() ) ); $log->info(" Waiting all logs to be applied on the latest slave.. "); $ret = $latest_slave->wait_until_relay_log_applied($log); if ($ret) { $log->error(" Failed with return code $ret"); return -1; } $latest_slave->current_slave_position(); $relay_master_log_file = $latest_slave->{Relay_Master_Log_File}; $exec_master_log_pos = $latest_slave->{Exec_Master_Log_Pos}; $ret = $_server_manager->change_master_and_start_slave( $target, $latest_slave, undef, undef, $log ); if ($ret) { $log->error(" Failed with return code $ret"); return -1; } $ret = $_server_manager->wait_until_in_sync( $target, $latest_slave ); if ($ret) { $log->error(" Failed with return code $ret"); return -1; } $log->info(" done."); } else { $target->current_slave_position(); $relay_master_log_file = $target->{Relay_Master_Log_File}; $exec_master_log_pos = $target->{Exec_Master_Log_Pos}; } if ( save_from_binlog_server( $relay_master_log_file, $exec_master_log_pos, $binlog_server_ref ) ) { apply_binlog_to_master($target); } return $_server_manager->get_new_master_binlog_position($target); } sub recover_master_internal($$) { my $target = shift; my $latest_slave = shift; $log->info(); $log->info("* Phase 3.4: New Master Diff Log Generation Phase..\n"); $log->info(); my $rc = recover_relay_logs( $target, $latest_slave ); if ( $rc && $rc != $GEN_DIFF_OK ) { return; } if ( send_binlog($target) ) { return; } $log->info(); $log->info("* Phase 3.5: Master Log Apply Phase..\n"); $log->info(); $log->info( "*NOTICE: If any error happens from this phase, manual recovery is needed." ); if ( recover_slave($target) ) { return; } return $_server_manager->get_new_master_binlog_position($target); } sub recover_master($$$$) { my $dead_master = shift; my $new_master = shift; my $latest_base_slave = shift; my $binlog_server_ref = shift; my ( $master_log_file, $master_log_pos, $exec_gtid_set ); if ( $_server_manager->is_gtid_auto_pos_enabled() ) { ( $master_log_file, $master_log_pos, $exec_gtid_set ) = recover_master_gtid_internal( $new_master, $latest_base_slave, $binlog_server_ref ); if ( !$exec_gtid_set ) { my $msg = "Recovering master server failed."; $log->error($msg); $mail_body .= $msg . "\n"; croak; } $log->info( sprintf( "Master Recovery succeeded. File:Pos:Exec_Gtid_Set: %s, %d, %s", $master_log_file, $master_log_pos, $exec_gtid_set ) ); } else { ( $master_log_file, $master_log_pos ) = recover_master_internal( $new_master, $latest_base_slave ); if ( !$master_log_file or !defined($master_log_pos) ) { my $msg = "Recovering master server failed."; # generating diff file failed: try to use other latest server # recoverable error on master: try to recover other master # unrecoverable error on master: destroying the master $log->error($msg); $mail_body .= $msg . "\n"; croak; } } $mail_body .= $new_master->get_hostinfo() . ": OK: Applying all logs succeeded.\n"; if ( $new_master->{master_ip_failover_script} ) { my $command = "$new_master->{master_ip_failover_script} --command=start --ssh_user=$new_master->{ssh_user} --orig_master_host=$dead_master->{hostname} --orig_master_ip=$dead_master->{ip} --orig_master_port=$dead_master->{port} --new_master_host=$new_master->{hostname} --new_master_ip=$new_master->{ip} --new_master_port=$new_master->{port} --new_master_user=$new_master->{escaped_user}"; $command .= $dead_master->get_ssh_args_if( 1, "orig", $_real_ssh_reachable ); $command .= $new_master->get_ssh_args_if( 2, "new", 1 ); $log->info("Executing master IP activate script:"); $log->info(" $command --new_master_password=xxx"); $command .= " --new_master_password=$new_master->{escaped_password}"; my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $g_logfile ); if ( $high == 0 && $low == 0 ) { $log->info(" OK."); $mail_body .= $new_master->get_hostinfo() . ": OK: Activated master IP address.\n"; } else { my $message = "Failed to activate master IP address for " . $new_master->get_hostinfo() . " with return code $high:$low"; $log->error( " " . $message ); $mail_body .= $message . "\n"; if ( $high == 10 ) { $log->warning("Proceeding."); } else { croak; } } } else { $log->warning( "master_ip_failover_script is not set. Skipping taking over new master IP address." ); } # Allow write access on master (if read_only==1) unless ($g_skip_disable_read_only) { $new_master->disable_read_only(); } $log->info("** Finished master recovery successfully."); $mail_subject .= " to " . $new_master->get_hostinfo(); return ( $master_log_file, $master_log_pos, $exec_gtid_set ); } sub recover_slaves_gtid_internal { my $new_master = shift; my $exec_gtid_set = shift; my @alive_slaves = $_server_manager->get_alive_slaves(); $log->info(); $log->info("* Phase 4.1: Starting Slaves in parallel..\n"); $log->info(); my $pm = new Parallel::ForkManager( $#alive_slaves + 1 ); my $slave_starting_fail = 0; $pm->run_on_start( sub { my ( $pid, $target ) = @_; $log->info( sprintf( "-- Slave recovery on host %s started, pid: %d. Check tmp log $g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log if it takes time..", $target->get_hostinfo(), $pid ) ); } ); $pm->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; $log->info(); $log->info("Log messages from $target->{hostname} ..."); my $local_file = "$g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log"; $log->info( "\n" . `cat $local_file` ); $log->info("End of log messages from $target->{hostname}."); unlink $local_file; if ( $exit_code == 0 ) { $target->{recover_ok} = 1; $log->info( sprintf( "-- Slave on host %s started.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": OK: Slave started, replicating from " . $new_master->get_hostinfo() . "\n"; } elsif ( $exit_code == 100 ) { $slave_starting_fail = 1; $mail_body .= $target->get_hostinfo() . ": ERROR: Starting slave failed.\n"; } elsif ( $exit_code == 120 ) { $slave_starting_fail = 1; $mail_body .= $target->get_hostinfo() . ": ERROR: Failed on waiting gtid exec set on master.\n"; } else { $slave_starting_fail = 1; $log->info( sprintf( "-- Recovery on host %s failed, exit code %d", $target->get_hostinfo(), $exit_code ) ); $mail_body .= $target->get_hostinfo() . ": ERROR: Starting slave failed.\n"; } } ); foreach my $target (@alive_slaves) { # master was already recovered next if ( $target->{id} eq $new_master->{id} ); my $pid = $pm->start($target) and next; my $pplog; eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; my $local_file = "$g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log"; unlink $local_file; $pplog = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); $pplog->add( Log::Dispatch::File->new( name => 'file', filename => $local_file, min_level => $target->{log_level}, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append' ) ); if ( $_server_manager->change_master_and_start_slave( $target, $new_master, undef, undef, $pplog ) ) { $pm->finish(100); } if ( $g_wait_until_gtid_in_sync && $target->gtid_wait( $exec_gtid_set, $pplog ) ) { $pm->finish(120); } else { $pm->finish(0); } }; if ($@) { $pplog->error($@) if ($pplog); undef $@; $pm->finish(1); } } $pm->wait_all_children; return ($slave_starting_fail); } sub recover_slaves_internal { my $new_master = shift; my $master_log_file = shift; my $master_log_pos = shift; my $latest_base_slave = shift; my @alive_slaves = $_server_manager->get_alive_slaves(); $log->info(); $log->info("* Phase 4.2: Starting Parallel Slave Log Apply Phase..\n"); $log->info(); # Recover other slaves # start slave if needed # Concurrency should be the number of alive slave servers -1. Sometimes the new master and the latest slave becomes the same machine. my $pm = new Parallel::ForkManager( $#alive_slaves + 1 ); my $skipping = 0; my $copy_fail = 0; my $recover_fail = 0; my $slave_starting_fail = 0; $pm->run_on_start( sub { my ( $pid, $target ) = @_; $log->info( sprintf( "-- Slave recovery on host %s started, pid: %d. Check tmp log $g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log if it takes time..", $target->get_hostinfo(), $pid ) ); } ); $pm->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; $log->info(); $log->info("Log messages from $target->{hostname} ..."); my $local_file = "$g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log"; $log->info( "\n" . `cat $local_file` ); $log->info("End of log messages from $target->{hostname}."); unlink $local_file; if ( $exit_code == 0 ) { $target->{recover_ok} = 1; $log->info( sprintf( "-- Slave recovery on host %s succeeded.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": OK: Applying all logs succeeded. Slave started, replicating from " . $new_master->get_hostinfo() . "\n"; } elsif ( $exit_code == 10 ) { $target->{recover_ok} = 1; $log->info( sprintf( "-- Slave recovery on host %s succeeded.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": OK: Applying all logs succeeded.\n"; } elsif ( $exit_code == 20 ) { $skipping = 1; $log->info( sprintf( "-- Skipping recovering slave %s because diff log generation failed.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": ERROR: Skipping applying logs because diff log generation failed.\n"; } elsif ( $exit_code == 30 ) { $copy_fail = 1; $log->info( sprintf( "-- Copying master binlog to host %s failed.", $target->get_hostinfo() ) ); $mail_body .= $target->get_hostinfo() . ": ERROR: Sending dead master's binlog failed.\n"; } elsif ( $exit_code == 100 ) { $slave_starting_fail = 1; $mail_body .= $target->get_hostinfo() . ": WARN: Applying all logs succeeded. But starting slave failed.\n"; } else { $recover_fail = 1; $log->info( sprintf( "-- Recovery on host %s failed, exit code %d", $target->get_hostinfo(), $exit_code ) ); $mail_body .= $target->get_hostinfo() . ": ERROR: Applying logs failed.\n"; } } ); foreach my $target (@alive_slaves) { # master was already recovered next if ( $target->{id} eq $new_master->{id} ); my $pid = $pm->start($target) and next; my $pplog; eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; my $local_file = "$g_workdir/$target->{hostname}_$target->{port}_$_start_datetime.log"; unlink $local_file; $pplog = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); $pplog->add( Log::Dispatch::File->new( name => 'file', filename => $local_file, min_level => $target->{log_level}, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append' ) ); unless ( $target->{gen_diff_ok} ) { $pm->finish(20); } if ( send_binlog( $target, $pplog ) ) { $pm->finish(30); } $target->current_slave_position(); if ( recover_slave( $target, $pplog ) ) { $pm->finish(1); } if ($g_skip_change_master) { $pplog->info("Skipping change master and start slave"); $pm->finish(10); } if ( $_server_manager->change_master_and_start_slave( $target, $new_master, $master_log_file, $master_log_pos, $pplog ) ) { $pm->finish(100); } else { $pm->finish(0); } }; if ($@) { $pplog->error($@) if ($pplog); undef $@; $pm->finish(1); } } $pm->wait_all_children; return ( $skipping || $copy_fail || $recover_fail || $slave_starting_fail ); } sub reconf_alive_servers { my $dead_master = shift; my @alive_slaves = $_server_manager->get_alive_slaves(); foreach my $slave (@alive_slaves) { next if ( $slave->{latest} ); next if ( $slave->{ssh_ok} < 2 ); next if ( $slave->{lack_relay_log} ); $slave->check_set_ssh_status( $log, 1 ); } my $init_again = 0; foreach my $slave (@alive_slaves) { if ( $slave->{ssh_ok} == 0 || $slave->{lack_relay_log} ) { $init_again = 1; last; } } if ($init_again) { $_server_manager->init_servers(); $log->info("Dead Servers:"); $_server_manager->print_dead_servers(); $log->info("Alive Slaves:"); $_server_manager->print_alive_slaves(); $_server_manager->print_failed_slaves_if(); $_server_manager->print_unmanaged_slaves_if(); } $_server_manager->validate_num_alive_servers( $dead_master, 1 ); } sub report_failed_slaves($) { my $dead_master = shift; my $has_failed_slaves = 0; my @dead_servers = $_server_manager->get_dead_servers(); foreach (@dead_servers) { next if ( $_->{id} eq $dead_master->{id} ); $mail_body .= $_->get_hostinfo() . ": ERROR: Could not be reachable so couldn't recover.\n"; $has_failed_slaves = 1; } my @failed_slaves = $_server_manager->get_failed_slaves(); foreach (@failed_slaves) { $mail_body .= $_->get_hostinfo() . ": ERROR: Slave failed so couldn't recover.\n"; $has_failed_slaves = 1; } return $has_failed_slaves; } sub recover_slaves($$$$$$) { my $dead_master = shift; my $new_master = shift; my $latest_base_slave = shift; my $master_log_file = shift; my $master_log_pos = shift; my $exec_gtid_set = shift; my $recover_slave_rc; if ( $_server_manager->is_gtid_auto_pos_enabled() ) { $recover_slave_rc = recover_slaves_gtid_internal( $new_master, $exec_gtid_set ); } else { if ( recover_all_slaves_relay_logs( $new_master, $latest_base_slave ) ) { my $msg = "Generating relay diff files from the latest slave failed."; $log->error($msg); $mail_body .= "$msg\n"; } else { my $msg = "Generating relay diff files from the latest slave succeeded."; $log->info($msg); $mail_body .= "$msg\n"; } $recover_slave_rc = recover_slaves_internal( $new_master, $master_log_file, $master_log_pos, $latest_base_slave ); } my $reset_slave_rc; if ( $recover_slave_rc == 0 ) { if ($g_skip_change_master) { $log->info("All slave servers are applied logs successfully."); $log->info(); } else { $log->info("All new slave servers recovered successfully."); $log->info(); $log->info("* Phase 5: New master cleanup phase.."); $log->info(); if ( $new_master->{skip_reset_slave} ) { $log->info("Skipping RESET SLAVE on the new master."); $reset_slave_rc = 0; } else { $log->info("Resetting slave info on the new master.."); $reset_slave_rc = $new_master->reset_slave_on_new_master(); if ( $reset_slave_rc eq '0' ) { $mail_body .= $new_master->get_hostinfo() . ": Resetting slave info succeeded.\n"; } else { $mail_body .= $new_master->get_hostinfo() . ": Resetting slave info failed.\n"; } } } } my $has_failed_servers = report_failed_slaves($dead_master); my $all_ok = 1; if ( $recover_slave_rc || $reset_slave_rc || $has_failed_servers ) { $all_ok = 0; } if ($all_ok) { $mail_subject .= " succeeded"; my $message = sprintf( "Master failover to %s completed successfully.", $new_master->get_hostinfo() ); $log->info($message); $mail_body .= $message . "\n"; return 0; } else { my $message = sprintf( "Master failover to %s done, but recovery on slave partially failed.", $new_master->get_hostinfo() ); $log->error($message); $mail_body .= $message . "\n"; return 10; } } sub cleanup { $_server_manager->release_failover_advisory_lock(); $_server_manager->disconnect_all(); MHA::NodeUtil::create_file_if($_failover_complete_file); $_create_error_file = 0; return 0; } sub send_report { my $dead_master = shift; my $new_master = shift; if ( $mail_subject && $mail_body ) { $log->info( "\n\n" . "----- Failover Report -----\n\n" . $mail_subject . "\n\n" . $mail_body ); if ( $dead_master->{report_script} ) { my $new_slaves = ""; my @alive_slaves = $_server_manager->get_alive_slaves(); foreach my $slave (@alive_slaves) { if ( $slave->{recover_ok} ) { $new_slaves .= "," if ($new_slaves); $new_slaves .= $slave->{hostname}; } } my $command = "$dead_master->{report_script} --orig_master_host=$dead_master->{hostname} "; if ( $new_master && $new_master->{hostname} && $new_master->{activated} ) { $command .= " --new_master_host=$new_master->{hostname} "; $command .= " --new_slave_hosts=$new_slaves "; } $command .= " --conf=$g_config_file "; $command .= " --subject=\"$mail_subject\" --body=\"$mail_body\""; $log->info("Sending mail.."); my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $g_logfile ); if ( $high != 0 || $low != 0 ) { $log->error("Failed to send mail with return code $high:$low"); } } } } sub do_master_failover { my $error_code = 1; my ( $dead_master, $new_master ); eval { my ( $servers_config_ref, $binlog_server_ref ) = init_config(); $log->info("Starting master failover."); $log->info(); $log->info("* Phase 1: Configuration Check Phase..\n"); $log->info(); MHA::ServerManager::init_binlog_server( $binlog_server_ref, $log ); $dead_master = check_settings($servers_config_ref); if ( $_server_manager->is_gtid_auto_pos_enabled() ) { $log->info("Starting GTID based failover."); } else { $_server_manager->force_disable_log_bin_if_auto_pos_disabled(); $log->info("Starting Non-GTID based failover."); } $log->info(); $log->info("** Phase 1: Configuration Check Phase completed.\n"); $log->info(); $log->info("* Phase 2: Dead Master Shutdown Phase..\n"); $log->info(); force_shutdown($dead_master); $log->info("* Phase 2: Dead Master Shutdown Phase completed.\n"); $log->info(); $log->info("* Phase 3: Master Recovery Phase..\n"); $log->info(); $log->info("* Phase 3.1: Getting Latest Slaves Phase..\n"); $log->info(); check_set_latest_slaves(); if ( !$_server_manager->is_gtid_auto_pos_enabled() ) { $log->info(); $log->info("* Phase 3.2: Saving Dead Master's Binlog Phase..\n"); $log->info(); save_master_binlog($dead_master); } $log->info(); $log->info("* Phase 3.3: Determining New Master Phase..\n"); $log->info(); my $latest_base_slave; if ( $_server_manager->is_gtid_auto_pos_enabled() ) { $latest_base_slave = $_server_manager->get_most_advanced_latest_slave(); } else { $latest_base_slave = find_latest_base_slave($dead_master); } $new_master = select_new_master( $dead_master, $latest_base_slave ); my ( $master_log_file, $master_log_pos, $exec_gtid_set ) = recover_master( $dead_master, $new_master, $latest_base_slave, $binlog_server_ref ); $new_master->{activated} = 1; $log->info("* Phase 3: Master Recovery Phase completed.\n"); $log->info(); $log->info("* Phase 4: Slaves Recovery Phase..\n"); $log->info(); $error_code = recover_slaves( $dead_master, $new_master, $latest_base_slave, $master_log_file, $master_log_pos, $exec_gtid_set ); if ( $g_remove_dead_master_conf && $error_code == 0 ) { MHA::Config::delete_block_and_save( $g_config_file, $dead_master->{id}, $log ); } cleanup(); }; if ($@) { if ( $dead_master && $dead_master->{not_error} ) { $log->info($@); } else { MHA::ManagerUtil::print_error( "Got ERROR: $@", $log ); $mail_body .= "Got Error so couldn't continue failover from here.\n" if ($mail_body); } $_server_manager->disconnect_all() if ($_server_manager); undef $@; } eval { send_report( $dead_master, $new_master ); MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ) unless ($error_code); if ($_create_error_file) { MHA::NodeUtil::create_file_if($_failover_error_file); } }; if ($@) { MHA::ManagerUtil::print_error( "Got ERROR on final reporting: $@", $log ); undef $@; } return $error_code; } sub finalize_on_error { eval { # Failover failure happened $_status_handler->update_status($MHA::ManagerConst::ST_FAILOVER_ERROR_S) if ($_status_handler); if ( $g_wait_on_failover_error > 0 && !$g_interactive ) { if ($log) { $log->info( "Waiting for $g_wait_on_failover_error seconds for error exit.."); } else { print "Waiting for $g_wait_on_failover_error seconds for error exit..\n"; } sleep $g_wait_on_failover_error; } MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ) if ($_status_handler); }; if ($@) { MHA::ManagerUtil::print_error( "Got Error on finalize_on_error at failover: $@", $log ); undef $@; } } sub main { local $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = \&exit_by_signal; local @ARGV = @_; my ( $master_host, $master_ip, $master_port, $error_code ); my ( $year, $mon, @time ) = reverse( (localtime)[ 0 .. 5 ] ); $_start_datetime = sprintf '%04d%02d%02d%02d%02d%02d', $year + 1900, $mon + 1, @time; GetOptions( 'global_conf=s' => \$g_global_config_file, 'conf=s' => \$g_config_file, 'dead_master_host=s' => \$master_host, 'dead_master_ip=s' => \$master_ip, 'dead_master_port=i' => \$master_port, 'new_master_host=s' => \$g_new_master_host, 'new_master_port=i' => \$g_new_master_port, 'interactive=i' => \$g_interactive, 'ssh_reachable=i' => \$g_ssh_reachable, 'last_failover_minute=i' => \$g_last_failover_minute, 'wait_on_failover_error=i' => \$g_wait_on_failover_error, 'ignore_last_failover' => \$g_ignore_last_failover, 'workdir=s' => \$g_workdir, 'manager_workdir=s' => \$g_workdir, 'log_output=s' => \$g_logfile, 'manager_log=s' => \$g_logfile, 'skip_save_master_binlog' => \$g_skip_save_master_binlog, 'remove_dead_master_conf' => \$g_remove_dead_master_conf, 'remove_orig_master_conf' => \$g_remove_dead_master_conf, 'skip_change_master' => \$g_skip_change_master, 'skip_disable_read_only' => \$g_skip_disable_read_only, 'wait_until_gtid_in_sync=i' => \$g_wait_until_gtid_in_sync, 'ignore_binlog_server_error' => \$g_ignore_binlog_server_error, ); setpgrp( 0, $$ ) unless ($g_interactive); unless ($g_config_file) { print "--conf= must be set.\n"; return 1; } unless ($master_host) { print "--dead_master_host= must be set.\n"; return 1; } unless ($master_ip) { $master_ip = MHA::NodeUtil::get_ip($master_host); print "--dead_master_ip= is not set. Using $master_ip.\n"; } unless ($master_port) { $master_port = 3306; print "--dead_master_port= is not set. Using $master_port.\n"; } $_dead_master_arg{hostname} = $master_host; $_dead_master_arg{ip} = $master_ip; $_dead_master_arg{port} = $master_port; # in interactive mode, always prints to stdout/stderr $g_logfile = undef if ($g_interactive); eval { $error_code = do_master_failover(); }; if ($@) { $error_code = 1; } if ($error_code) { finalize_on_error(); } return $error_code; } 1; mha4mysql-manager-0.58/lib/MHA/MasterMonitor.pm000066400000000000000000000561461325502577300213360ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::MasterMonitor; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use English qw(-no_match_vars); use Getopt::Long qw(:config pass_through); use Pod::Usage; use Log::Dispatch; use Log::Dispatch::Screen; use MHA::Config; use MHA::ServerManager; use MHA::HealthCheck; use MHA::FileStatus; use MHA::SSHCheck; use MHA::ManagerConst; use MHA::ManagerUtil; use MHA::BinlogManager; use File::Basename; my $g_global_config_file = $MHA::ManagerConst::DEFAULT_GLOBAL_CONF; my $g_config_file; my $g_check_only; my $g_check_repl_health; my $g_seconds_behind_master = 30; my $g_monitor_only; my $g_workdir; my $g_interactive = 1; my $g_logfile; my $g_wait_on_monitor_error = 0; my $g_skip_ssh_check; my $g_ignore_fail_on_start = 0; my $_master_node_version; my $_server_manager; my $_master_ping; my $RETRY = 100; my $_status_handler; my $log; sub exit_by_signal { $log->info("Got terminate signal. Exit."); eval { MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ) unless ($g_check_only); $_server_manager->disconnect_all() if ($_server_manager); $_master_ping->disconnect_if() if ($_master_ping); }; if ($@) { $log->error("Got Error: $@"); undef $@; } exit 1; } sub get_binlog_check_command { my $target = shift; my $use_prefix = shift; # this file is not created. just checking directory path my $workfile = "$target->{remote_workdir}/save_binary_logs_test"; my $command = "save_binary_logs --command=test --start_pos=4 --binlog_dir=$target->{master_binlog_dir} --output_file=$workfile --manager_version=$MHA::ManagerConst::VERSION"; my $file; if ( $target->{File} ) { $file = $target->{File}; } else { my @alive_slaves = $_server_manager->get_alive_slaves(); my $slave = $alive_slaves[0]; $slave->current_slave_position(); $file = $slave->{Master_Log_File}; } if ($use_prefix) { my ( $binlog_prefix, $number ) = MHA::BinlogManager::get_head_and_number($file); $command .= " --binlog_prefix=$binlog_prefix"; } else { $command .= " --start_file=$file"; } unless ( $target->{handle_raw_binlog} ) { my $oldest_version = $_server_manager->get_oldest_version(); $command .= " --oldest_version=$oldest_version "; } if ( $target->{log_level} eq "debug" ) { $command .= " --debug "; } return $command; } sub check_master_ssh_env($) { my $target = shift; $log->info( "Checking SSH publickey authentication settings on the current master.."); my $ssh_reachable; if ( MHA::HealthCheck::ssh_check_simple( $target->{ssh_user}, $target->{ssh_host}, $target->{ssh_ip}, $target->{ssh_port}, $target->{logger}, $target->{ssh_connection_timeout} ) ) { $ssh_reachable = 0; } else { $ssh_reachable = 1; } if ( $ssh_reachable && !$target->{has_gtid} ) { $_master_node_version = MHA::ManagerUtil::get_node_version( $log, $target->{ssh_user}, $target->{ssh_host}, $target->{ssh_ip}, $target->{ssh_port} ); if ( !$_master_node_version ) { $log->error( "Failed to get MHA node version on the current master even though current master is reachable via SSH!" ); croak; } $log->info("Master MHA Node version is $_master_node_version."); } return $ssh_reachable; } sub check_binlog($) { my $target = shift; $log->info( sprintf( "Checking recovery script configurations on %s..", $target->get_hostinfo() ) ); my $ssh_user_host = $target->{ssh_user} . '@' . $target->{ssh_ip}; my $command = get_binlog_check_command($target); $log->info(" Executing command: $command "); $log->info( " Connecting to $ssh_user_host($target->{ssh_host}:$target->{ssh_port}).. " ); my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $target->{ssh_port}, $command, $g_logfile ); if ( $high ne '0' || $low ne '0' ) { $log->error("Binlog setting check failed!"); return 1; } $log->info("Binlog setting check done."); return 0; } sub check_slave_env() { my @alive_servers = $_server_manager->get_alive_slaves(); $log->info( "Checking SSH publickey authentication and checking recovery script configurations on all alive slave servers.." ); foreach my $s (@alive_servers) { my $ssh_user_host = $s->{ssh_user} . '@' . $s->{ssh_ip}; my $command = "apply_diff_relay_logs --command=test --slave_user=$s->{escaped_user} --slave_host=$s->{hostname} --slave_ip=$s->{ip} --slave_port=$s->{port} --workdir=$s->{remote_workdir} --target_version=$s->{mysql_version} --manager_version=$MHA::ManagerConst::VERSION"; if ( $s->{client_bindir} ) { $command .= " --client_bindir=$s->{client_bindir}"; } if ( $s->{client_libdir} ) { $command .= " --client_libdir=$s->{client_libdir}"; } if ( $s->{relay_log_info_type} eq "TABLE" ) { $command .= " --relay_dir=$s->{relay_dir} --current_relay_log=$s->{current_relay_log} "; } else { $command .= " --relay_log_info=$s->{relay_log_info} "; $command .= " --relay_dir=$s->{datadir} "; } if ( $s->{log_level} eq "debug" ) { $command .= " --debug "; } if ($MHA::ManagerConst::USE_SSH_OPTIONS) { $command .= " --ssh_options='$MHA::NodeConst::SSH_OPT_ALIVE' "; } $log->info(" Executing command : $command --slave_pass=xxx"); if ( $s->{escaped_password} ne "" ) { $command .= " --slave_pass=$s->{escaped_password}"; } $log->info( " Connecting to $ssh_user_host($s->{ssh_host}:$s->{ssh_port}).. "); my ( $high, $low ) = MHA::ManagerUtil::exec_ssh_cmd( $ssh_user_host, $s->{ssh_port}, $command, $g_logfile ); if ( $high ne '0' || $low ne '0' ) { $log->error("Slaves settings check failed!"); return 1; } } $log->info("Slaves settings check done."); return 0; } sub check_scripts($) { my $current_master = shift; if ( $current_master->{master_ip_failover_script} ) { my $command = "$current_master->{master_ip_failover_script} --command=status --ssh_user=$current_master->{ssh_user} --orig_master_host=$current_master->{hostname} --orig_master_ip=$current_master->{ip} --orig_master_port=$current_master->{port}"; $command .= $current_master->get_ssh_args_if( 1, "orig", 1 ); $log->info("Checking master_ip_failover_script status:"); $log->info(" $command"); my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $g_logfile ); if ( $high == 0 && $low == 0 ) { $log->info(" OK."); } else { $log->error( " Failed to get master_ip_failover_script status with return code $high:$low." ); croak; } } else { $log->warning("master_ip_failover_script is not defined."); } if ( $current_master->{shutdown_script} ) { my $command = "$current_master->{shutdown_script} --command=status --ssh_user=$current_master->{ssh_user} --host=$current_master->{hostname} --ip=$current_master->{ip}"; $command .= $current_master->get_ssh_args_if( 1, "shutdown", 1 ); $log->info("Checking shutdown script status:"); $log->info(" $command"); my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $g_logfile ); if ( $high == 0 && $low == 0 ) { $log->info(" OK."); } else { $log->error(" Failed to get power status with return code $high:$low."); croak; } } else { $log->warning("shutdown_script is not defined."); } } sub check_binlog_servers { my $binlog_server_ref = shift; my $log = shift; my @binlog_servers = @$binlog_server_ref; if ( $#binlog_servers >= 0 ) { MHA::ServerManager::init_binlog_server( $binlog_server_ref, $log ); foreach my $server (@binlog_servers) { if ( check_binlog($server) ) { $log->error("Binlog server configuration failed."); croak; } } } } sub wait_until_master_is_unreachable() { my ( @servers_config, @servers, @dead_servers, @alive_servers, @alive_slaves, $current_master, $ret, $ssh_reachable ); my $func_rc = 1; eval { $g_logfile = undef if ($g_check_only); $log = MHA::ManagerUtil::init_log($g_logfile); unless ( -f $g_config_file ) { $log->error("Configuration file $g_config_file not found!"); croak; } my ( $sc_ref, $binlog_server_ref ) = new MHA::Config( logger => $log, globalfile => $g_global_config_file, file => $g_config_file )->read_config(); @servers_config = @$sc_ref; if ( !$g_logfile && !$g_check_only && $servers_config[0]->{manager_log} ) { $g_logfile = $servers_config[0]->{manager_log}; } $log = MHA::ManagerUtil::init_log( $g_logfile, $servers_config[0]->{log_level} ); $log->info("MHA::MasterMonitor version $MHA::ManagerConst::VERSION."); unless ($g_workdir) { if ( $servers_config[0]->{manager_workdir} ) { $g_workdir = $servers_config[0]->{manager_workdir}; } else { $g_workdir = "/var/tmp"; } } MHA::ManagerUtil::check_node_version($log); MHA::NodeUtil::check_manager_version($MHA::ManagerConst::VERSION); MHA::NodeUtil::create_dir_if($g_workdir); unless ($g_check_only) { $_status_handler = new MHA::FileStatus( conffile => $g_config_file, dir => $g_workdir ); $_status_handler->init(); if ( -f $_status_handler->{status_file} ) { $log->warning( "$_status_handler->{status_file} already exists. You might have killed manager with SIGKILL(-9), may run two or more monitoring process for the same application, or use the same working directory. Check for details, and consider setting --workdir separately." ); MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ); } $_status_handler->update_status( $MHA::ManagerConst::ST_INITIALIZING_MONITOR_S); } $_server_manager = new MHA::ServerManager( servers => \@servers_config ); $_server_manager->set_logger($log); $_server_manager->connect_all_and_read_server_status(); @dead_servers = $_server_manager->get_dead_servers(); @alive_servers = $_server_manager->get_alive_servers(); @alive_slaves = $_server_manager->get_alive_slaves(); $log->info("Dead Servers:"); $_server_manager->print_dead_servers(); $log->info("Alive Servers:"); $_server_manager->print_alive_servers(); $log->info("Alive Slaves:"); $_server_manager->print_alive_slaves(); $_server_manager->print_failed_slaves_if(); $_server_manager->print_unmanaged_slaves_if(); $current_master = $_server_manager->get_current_alive_master(); unless ($current_master) { if ($g_interactive) { print "Master is not currently alive. Proceed? (yes/NO): "; my $ret = ; chomp($ret); die "abort" if ( lc($ret) !~ /^y/ ); } } if ( $_server_manager->validate_slaves( $servers_config[0]->{check_repl_filter}, $current_master ) ) { $log->error("Slave configurations is not valid."); croak; } my @bad = $_server_manager->get_bad_candidate_masters(); if ( $#alive_slaves <= $#bad ) { $log->error( "None of slaves can be master. Check failover " . "configuration file or log-bin settings in my.cnf" ); croak; } $_server_manager->check_repl_priv(); if ( !$_server_manager->is_gtid_auto_pos_enabled() ) { $log->info("GTID (with auto-pos) is not supported"); MHA::SSHCheck::do_ssh_connection_check( \@alive_servers, $log, $servers_config[0]->{log_level}, $g_workdir ) unless ($g_skip_ssh_check); $log->info("Checking MHA Node version.."); foreach my $slave (@alive_slaves) { MHA::ManagerUtil::check_node_version( $log, $slave->{ssh_user}, $slave->{ssh_host}, $slave->{ssh_ip}, $slave->{ssh_port} ); } $log->info(" Version check ok."); } else { $log->info( "GTID (with auto-pos) is supported. Skipping all SSH and Node package checking." ); check_binlog_servers( $binlog_server_ref, $log ); } unless ($current_master) { $log->info("Getting current master (maybe dead) info .."); $current_master = $_server_manager->get_orig_master(); if ( !$current_master ) { $log->error("Failed to get current master info!"); croak; } $log->info( sprintf( "Identified master is %s.", $current_master->get_hostinfo() ) ); } $_server_manager->validate_num_alive_servers( $current_master, $g_ignore_fail_on_start ); if ( check_master_ssh_env($current_master) ) { if ( !$_server_manager->is_gtid_auto_pos_enabled() && check_binlog($current_master) ) { $log->error("Master configuration failed."); croak; } } $_status_handler->set_master_host( $current_master->{hostname} ) unless ($g_check_only); if ( !$_server_manager->is_gtid_auto_pos_enabled() && check_slave_env() ) { $log->error("Slave configuration failed."); croak; } $_server_manager->print_servers_ascii($current_master); $_server_manager->check_replication_health($g_seconds_behind_master) if ($g_check_repl_health); check_scripts($current_master); $_server_manager->disconnect_all(); $func_rc = 0; }; if ($@) { $log->error("Error happened on checking configurations. $@") if ($log); undef $@; return $func_rc; } return $func_rc if ($g_check_only); # master ping. This might take hours/days/months.. $func_rc = 1; eval { my $ssh_check_command; if (!$_server_manager->is_gtid_auto_pos_enabled() && $_master_node_version && $_master_node_version >= 0.53 ) { $ssh_check_command = get_binlog_check_command( $current_master, 1 ); } else { $ssh_check_command = "exit 0"; } $log->debug("SSH check command: $ssh_check_command"); $_master_ping = new MHA::HealthCheck( user => $current_master->{user}, password => $current_master->{password}, ip => $current_master->{ip}, hostname => $current_master->{hostname}, port => $current_master->{port}, interval => $current_master->{ping_interval}, ssh_user => $current_master->{ssh_user}, ssh_host => $current_master->{ssh_host}, ssh_ip => $current_master->{ssh_ip}, ssh_port => $current_master->{ssh_port}, ssh_connection_timeout => $current_master->{ssh_connection_timeout}, ssh_check_command => $ssh_check_command, status_handler => $_status_handler, logger => $log, logfile => $g_logfile, workdir => $g_workdir, ping_type => $current_master->{ping_type}, ); $log->info( sprintf( "Set master ping interval %d seconds.", $_master_ping->get_ping_interval() ) ); if ( $current_master->{secondary_check_script} ) { $_master_ping->set_secondary_check_script( $current_master->{secondary_check_script} ); $log->info( sprintf( "Set secondary check script: %s", $_master_ping->get_secondary_check_script() ) ); } else { $log->warning( "secondary_check_script is not defined. It is highly recommended setting it to check master reachability from two or more routes." ); } $log->info( sprintf( "Starting ping health check on %s..", $current_master->get_hostinfo() ) ); ( $ret, $ssh_reachable ) = $_master_ping->wait_until_unreachable(); if ( $ret eq '2' ) { $log->error( "Target master's advisory lock is already held by someone. Please check whether you monitor the same master from multiple monitoring processes." ); croak; } elsif ( $ret ne '0' ) { croak; } $log->warning( sprintf( "Master %s is not reachable!", $current_master->get_hostinfo() ) ); if ($ssh_reachable) { $log->warning("SSH is reachable."); } else { $log->warning("SSH is NOT reachable."); } $func_rc = 0; }; if ($@) { $log->error("Error happened on health checking. $@"); undef $@; return $func_rc; } $_status_handler->update_status($MHA::ManagerConst::ST_PING_FAILED_S); return ( $func_rc, $current_master, $ssh_reachable ); } sub wait_until_master_is_dead { my $exit_code = 1; my ( $ret, $dead_master, $ssh_reachable ) = wait_until_master_is_unreachable(); if ( !defined($ret) || $ret ne '0' ) { $log->error("Error happened on monitoring servers."); return $exit_code; } if ($g_check_only) { return 0; } # this should not happen unless ($dead_master) { $log->error("Dead master not found!\n"); return $exit_code; } # Master fails! # Reading config file and connecting to all hosts except master again # to check current availability $exit_code = eval { $log->info( "Connecting to a master server failed. Reading configuration " . "file $g_global_config_file and $g_config_file again, and trying to connect to all servers to " . "check server status.." ); my $conf = new MHA::Config( logger => $log, globalfile => $g_global_config_file, file => $g_config_file ); my ( $sc_ref, $binlog_server_ref ) = $conf->read_config(); my @servers_config = @$sc_ref; $_server_manager = new MHA::ServerManager( servers => \@servers_config ); $_server_manager->set_logger($log); $log->debug( sprintf( "Skipping connecting to dead master %s.", $dead_master->get_hostinfo() ) ); $_server_manager->connect_all_and_read_server_status( $dead_master->{hostname}, $dead_master->{ip}, $dead_master->{port} ); my @dead_servers = $_server_manager->get_dead_servers(); my @alive_servers = $_server_manager->get_alive_servers(); $log->info("Dead Servers:"); $_server_manager->print_dead_servers(); $log->info("Alive Servers:"); $_server_manager->print_alive_servers(); $log->info("Alive Slaves:"); $_server_manager->print_alive_slaves(); $_server_manager->print_failed_slaves_if(); $_server_manager->print_unmanaged_slaves_if(); my $real_master = $_server_manager->get_orig_master(); if ( $dead_master->{id} ne $real_master->{id} ) { $log->error( sprintf( "Monitor detected %s failed, but actual master server is %s. Check replication configurations again.", $dead_master->get_hostinfo(), $real_master->get_hostinfo() ) ); return 1; } # When this condition is met, master is actually alive. if ( $_server_manager->get_alive_server_by_id( $dead_master->{id} ) ) { $log->warning("master is actually alive. starting monitoring again."); return $RETRY; } if ( $_server_manager->validate_slaves( $servers_config[0]->{check_repl_filter} ) ) { $log->error( "At least one alive slave is not correctly configured. " . "Can't execute failover" ); return 1; } $log->info("Master is down!"); $log->info("Terminating monitoring script."); $_server_manager->disconnect_all() if ($_server_manager); return $MHA::ManagerConst::MASTER_DEAD_RC; }; if ($@) { $log->warning("Got Error: $@"); undef $@; $exit_code = 1; } return 1 if ( !defined($exit_code) ); return $MHA::ManagerConst::MASTER_DEAD_RC, $dead_master, $ssh_reachable if ( $exit_code == $MHA::ManagerConst::MASTER_DEAD_RC ); return $exit_code; } sub prepare_for_retry { eval { $_status_handler->update_status($MHA::ManagerConst::ST_RETRYING_MONITOR_S); $log->info("Waiting for $g_wait_on_monitor_error seconds for retrying.."); sleep $g_wait_on_monitor_error; MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ); }; if ($@) { MHA::ManagerUtil::print_error( "Got Error on prepare_for_retry at monitor: $@", $log ); undef $@; } } sub finalize_on_error { eval { # Monitor failure happened $_status_handler->update_status($MHA::ManagerConst::ST_CONFIG_ERROR_S) if ($_status_handler); if ( $g_wait_on_monitor_error > 0 ) { $log->info( "Waiting for $g_wait_on_monitor_error seconds for error exit.."); sleep $g_wait_on_monitor_error; } MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ) if ($_status_handler); }; if ($@) { MHA::ManagerUtil::print_error( "Got Error on finalize_on_error at monitor: $@", $log ); undef $@; } } sub finalize { eval { MHA::NodeUtil::drop_file_if( $_status_handler->{status_file} ) if ($_status_handler); }; if ($@) { MHA::ManagerUtil::print_error( "Got Error on finalize at monitor: $@", $log ); undef $@; } } sub main { local $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = \&exit_by_signal; local @ARGV = @_; GetOptions( 'global_conf=s' => \$g_global_config_file, 'conf=s' => \$g_config_file, 'check_only' => \$g_check_only, 'check_repl_health' => \$g_check_repl_health, 'seconds_behind_master=i' => \$g_seconds_behind_master, 'monitor_only' => \$g_monitor_only, 'interactive=i' => \$g_interactive, 'wait_on_monitor_error=i' => \$g_wait_on_monitor_error, 'workdir=s' => \$g_workdir, 'manager_workdir=s' => \$g_workdir, 'log_output=s' => \$g_logfile, 'manager_log=s' => \$g_logfile, 'skip_ssh_check' => \$g_skip_ssh_check, # for testing 'skip_check_ssh' => \$g_skip_ssh_check, 'ignore_fail_on_start' => \$g_ignore_fail_on_start, ); setpgrp( 0, $$ ) unless ($g_interactive); unless ($g_config_file) { print "--conf= must be set.\n"; return 1; } while (1) { my ( $exit_code, $dead_master, $ssh_reachable ) = wait_until_master_is_dead(); my $msg = sprintf( "Got exit code %d (%s).", $exit_code, $exit_code == $MHA::ManagerConst::MASTER_DEAD_RC ? "Master dead" : "Not master dead" ); $log->info($msg) if ($log); if ($g_check_only) { finalize(); return $exit_code; } if ( $exit_code && $exit_code == $RETRY ) { prepare_for_retry(); } else { if ( $exit_code && $exit_code != $MHA::ManagerConst::MASTER_DEAD_RC ) { finalize_on_error(); } elsif ($g_monitor_only) { finalize(); } return ( $exit_code, $dead_master, $ssh_reachable ); } } } 1; mha4mysql-manager-0.58/lib/MHA/MasterRotate.pm000066400000000000000000000560111325502577300211340ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::MasterRotate; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Carp qw(croak); use POSIX qw(:signal_h); use Getopt::Long; use Pod::Usage; use Log::Dispatch; use Log::Dispatch::Screen; use Log::Dispatch::File; use MHA::Config; use MHA::ServerManager; use MHA::Server; use MHA::ManagerUtil; use File::Basename; use Parallel::ForkManager; my $g_global_config_file = $MHA::ManagerConst::DEFAULT_GLOBAL_CONF; my $g_config_file; my $g_check_only; my $g_new_master_host; my $g_new_master_port = 3306; my $g_workdir; my $g_flush_tables = 2; my $g_orig_master_is_new_slave; my $g_running_updates_limit = 1; my $g_running_seconds_limit = 10; my $g_seconds_behind_master = 30; my $g_skip_lock_all_tables; my $g_remove_orig_master_conf; my $g_interactive = 1; my $_server_manager; my $start_datetime; my $log = MHA::ManagerUtil::init_log(); sub identify_orig_master() { my $orig_master; my ( @servers, @dead_servers, @alive_servers, @alive_slaves ); $log->info("MHA::MasterRotate version $MHA::ManagerConst::VERSION."); $log->info("Starting online master switch.."); $log->info(); $log->info("* Phase 1: Configuration Check Phase..\n"); $log->info(); my ( $sc_ref, undef ) = new MHA::Config( logger => $log, globalfile => $g_global_config_file, file => $g_config_file )->read_config(); my @servers_config = @$sc_ref; $log = MHA::ManagerUtil::init_log( undef, $servers_config[0]->{log_level} ); unless ($g_workdir) { if ( $servers_config[0]->{manager_workdir} ) { $g_workdir = $servers_config[0]->{manager_workdir}; } else { $g_workdir = "/var/tmp"; } } $_server_manager = new MHA::ServerManager( servers => \@servers_config ); $_server_manager->set_logger($log); $_server_manager->connect_all_and_read_server_status(); @servers = $_server_manager->get_servers(); @dead_servers = $_server_manager->get_dead_servers(); @alive_servers = $_server_manager->get_alive_servers(); @alive_slaves = $_server_manager->get_alive_slaves(); #Make sure that currently there is not any dead server. if ( $#dead_servers >= 0 ) { $log->error( "Switching master should not be started if one or more servers is down." ); $log->info("Dead Servers:"); $_server_manager->print_dead_servers(); croak; } $orig_master = $_server_manager->get_current_alive_master(); if ( !$orig_master ) { $log->error( "Failed to get current master. Maybe replication is misconfigured or configuration file is broken?" ); croak; } $log->info("Alive Slaves:"); $_server_manager->print_alive_slaves(1); $_server_manager->print_unmanaged_slaves_if(); $_server_manager->check_repl_priv(); my $run_flush_tables = 1; if ( $g_interactive && $g_flush_tables == 2 ) { printf( "\nIt is better to execute FLUSH NO_WRITE_TO_BINLOG TABLES on the master before switching. Is it ok to execute on %s? (YES/no): ", $orig_master->get_hostinfo() ); my $ret = ; chomp($ret); if ( lc($ret) !~ /^n/ ) { $run_flush_tables = 1; } else { $run_flush_tables = 0; } } else { $run_flush_tables = $g_flush_tables; } if ($run_flush_tables) { $orig_master->flush_tables(); } else { $log->info("Skipping executing FLUSH NO_WRITE_TO_BINLOG TABLES."); } $log->info("Checking MHA is not monitoring or doing failover.."); if ( $orig_master->get_monitor_advisory_lock(0) ) { $log->error( "Getting advisory lock failed on the current master. MHA Monitor runs on the current master. Stop MHA Manager/Monitor and try again." ); croak; } foreach my $target (@alive_slaves) { if ( $target->get_failover_advisory_lock(0) ) { $log->error( "Getting advisory lock failed on $target->{hostname}. Maybe failover script or purge_relay_logs script is running on the same slave?" ); croak; } } $_server_manager->check_replication_health($g_seconds_behind_master); my @threads = $orig_master->get_running_update_threads( $g_running_updates_limit + 1 ); if ( $#threads >= 0 ) { $log->error( sprintf( "We should not start online master switch when one of connections are running long updates on the current master(%s). Currently %d update thread(s) are running.", $orig_master->get_hostinfo(), $#threads + 1 ) ); MHA::DBHelper::print_threads_util( \@threads, 10 ); croak; } return $orig_master; } sub check_filter { my $orig_master = shift; my $new_master = shift; $orig_master->{Binlog_Do_DB} ||= ""; $new_master->{Binlog_Do_DB} ||= ""; $orig_master->{Binlog_Ignore_DB} ||= ""; $new_master->{Binlog_Ignore_DB} ||= ""; if ( $orig_master->{Binlog_Do_DB} ne $new_master->{Binlog_Do_DB} || $orig_master->{Binlog_Ignore_DB} ne $new_master->{Binlog_Ignore_DB} ) { $log->error( "Binlog filtering check failed on the new master! Orig master and New master must have same binlog filtering rules (same binlog-do-db and binlog-ignore-db). Check SHOW MASTER STATUS output and set my.cnf correctly." ); my $msg = "Bad Binlog/Replication filtering rules:\n"; $msg .= $orig_master->print_filter( 1, 0 ); $msg .= $new_master->print_filter( 0, 0 ); $log->warning($msg); croak; } if ($g_orig_master_is_new_slave) { $orig_master->read_repl_filter(); $orig_master->{Replicate_Do_Table} ||= ""; $orig_master->{Replicate_Ignore_Table} ||= ""; $orig_master->{Replicate_Wild_Do_Table} ||= ""; $orig_master->{Replicate_Wild_Ignore_Table} ||= ""; $new_master->{Replicate_Do_Table} ||= ""; $new_master->{Replicate_Ignore_Table} ||= ""; $new_master->{Replicate_Wild_Do_Table} ||= ""; $new_master->{Replicate_Wild_Ignore_Table} ||= ""; if ( $orig_master->{Replicate_Do_Table} ne $new_master->{Replicate_Do_Table} || $orig_master->{Replicate_Ignore_Table} ne $new_master->{Replicate_Ignore_Table} || $orig_master->{Replicate_Wild_Do_Table} ne $new_master->{Replicate_Wild_Do_Table} || $orig_master->{Replicate_Wild_Ignore_Table} ne $new_master->{Replicate_Wild_Ignore_Table} ) { $log->error( "Replication filtering check failed on the orig/new master! Orig master and New master must have same replication filtering rules --orig_master_is_new_slave is set. Check SHOW SLAVE STATUS output and/or set my.cnf correctly." ); my $msg = "Bad Binlog/Replication filtering rules:\n"; $msg .= $orig_master->print_filter( 1, 1 ); $msg .= $new_master->print_filter( 0, 1 ); $log->warning($msg); croak; } } } sub identify_new_master { my $orig_master = shift; $_server_manager->set_latest_slaves( $_server_manager->{alive_slaves} ); my $new_master = $_server_manager->select_new_master( $g_new_master_host, $g_new_master_port, 0 ); unless ($new_master) { $log->error("Failed to get new master!"); croak; } $_server_manager->print_servers_migration_ascii( $orig_master, $new_master, $g_orig_master_is_new_slave ); if ( $g_interactive && !$g_check_only ) { $new_master = $_server_manager->manually_decide_new_master( $orig_master, $new_master ); } $log->info( sprintf( "Checking whether %s is ok for the new master..", $new_master->get_hostinfo() ) ); if ( $_server_manager->is_target_bad_for_new_master($new_master) ) { $log->error( sprintf( "Server %s is not correctly configured to be new master!", $new_master->get_hostinfo() ) ); die; } $log->info(" ok."); if ( $orig_master->{check_repl_filter} ) { check_filter( $orig_master, $new_master ); } my @threads = $new_master->get_running_threads($g_running_seconds_limit); if ( $#threads >= 0 ) { $log->error( sprintf( "We should not start online master switch when one of connections are running long queries on the new master(%s). Currently %d thread(s) are running.", $new_master->get_hostinfo(), $#threads + 1 ) ); MHA::DBHelper::print_threads_util( \@threads, 10 ); croak; } return $new_master; } sub reject_update($$) { my $orig_master = shift; my $new_master = shift; my $ret; $orig_master->release_monitor_advisory_lock(); $orig_master->disconnect(); $log->info("* Phase 2: Rejecting updates Phase..\n"); $log->info(); if ( $new_master->{master_ip_online_change_script} ) { my $command = "$orig_master->{master_ip_online_change_script} --command=stop --orig_master_host=$orig_master->{hostname} --orig_master_ip=$orig_master->{ip} --orig_master_port=$orig_master->{port} --orig_master_user=$orig_master->{escaped_user} --new_master_host=$new_master->{hostname} --new_master_ip=$new_master->{ip} --new_master_port=$new_master->{port} --new_master_user=$new_master->{escaped_user}"; $command .= " --orig_master_ssh_user=$orig_master->{ssh_user}"; $command .= " --new_master_ssh_user=$new_master->{ssh_user}"; $command .= $orig_master->get_ssh_args_if( 1, "orig", 1 ); $command .= $new_master->get_ssh_args_if( 2, "new", 1 ); if ($g_orig_master_is_new_slave) { $command .= " --orig_master_is_new_slave"; } $log->info( "Executing master ip online change script to disable write on the current master:" ); $log->info( " $command --orig_master_password=xxx --new_master_password=xxx"); $command .= " --orig_master_password=$orig_master->{escaped_password} --new_master_password=$new_master->{escaped_password}"; my ( $high, $low ) = MHA::ManagerUtil::exec_system($command); if ( $high == 0 && $low == 0 ) { $log->info(" ok."); } else { if ( $high == 10 ) { $log->warning("Proceeding."); } else { croak; } } } elsif ($g_interactive) { print "master_ip_online_change_script is not defined. If you do not disable writes on the current master manually, applications keep writing on the current master. Is it ok to proceed? (yes/NO): "; $ret = ; chomp($ret); if ( lc($ret) !~ /^y/ ) { $orig_master->{not_error} = 1; die "Not typed yes. Stopping."; } } else { $log->warning( "master_ip_online_change_script is not defined. Skipping disabling writes on the current master." ); } # It is necessary to keep connecting on the orig master to check # binary log is not proceeding. master write control script may kill # previous connections, so it is needed to establish connection again. unless ( $orig_master->reconnect() ) { $log->error( sprintf( "Failed to connect to the orig master %s!", $orig_master->get_hostinfo() ) ); croak; } my ( $orig_master_log_file, $orig_master_log_pos ); if ($g_skip_lock_all_tables) { $log->info("Skipping locking all tables."); } else { $log->info( "Locking all tables on the orig master to reject updates from everybody (including root):" ); } if ( $g_skip_lock_all_tables || $orig_master->lock_all_tables() ) { # FLUSH TABLES WITH READ LOCK is skipped or failed. # So we need to verify binlog writes are stopped or not. # All slaves should complete replication until this position ( $orig_master_log_file, $orig_master_log_pos ) = $orig_master->check_binlog_stop(); } else { ( $orig_master_log_file, $orig_master_log_pos ) = $orig_master->get_binlog_position(); } if ( !$orig_master_log_file || !defined($orig_master_log_pos) ) { $log->error("Failed to get orig master's binlog and/or position!"); croak; } $log->info( sprintf( "Orig master binlog:pos is %s:%d.", $orig_master_log_file, $orig_master_log_pos ) ); return ( $orig_master_log_file, $orig_master_log_pos ); } sub switch_master_internal($$$) { my $target = shift; my $orig_master_log_file = shift; my $orig_master_log_pos = shift; if ( $target->master_pos_wait( $orig_master_log_file, $orig_master_log_pos ) ) { return; } return $_server_manager->get_new_master_binlog_position($target); } sub switch_master($$$$) { my $orig_master = shift; my $new_master = shift; my $orig_master_log_file = shift; my $orig_master_log_pos = shift; my ( $master_log_file, $master_log_pos ) = switch_master_internal( $new_master, $orig_master_log_file, $orig_master_log_pos ); if ( !$master_log_file or !defined($master_log_pos) ) { $log->error("Failed to get new master's binlog and/or position!"); croak; } # Allow write access on the new master if ( $new_master->{master_ip_online_change_script} ) { my $command = "$new_master->{master_ip_online_change_script} --command=start --orig_master_host=$orig_master->{hostname} --orig_master_ip=$orig_master->{ip} --orig_master_port=$orig_master->{port} --orig_master_user=$orig_master->{escaped_user} --new_master_host=$new_master->{hostname} --new_master_ip=$new_master->{ip} --new_master_port=$new_master->{port} --new_master_user=$new_master->{escaped_user}"; $command .= " --orig_master_ssh_user=$orig_master->{ssh_user}"; $command .= " --new_master_ssh_user=$new_master->{ssh_user}"; $command .= $orig_master->get_ssh_args_if( 1, "orig", 1 ); $command .= $new_master->get_ssh_args_if( 2, "new", 1 ); if ($g_orig_master_is_new_slave) { $command .= " --orig_master_is_new_slave"; } $log->info( "Executing master ip online change script to allow write on the new master:" ); $log->info( " $command --orig_master_password=xxx --new_master_password=xxx"); $command .= " --orig_master_password=$orig_master->{escaped_password} --new_master_password=$new_master->{escaped_password}"; my ( $high, $low ) = MHA::ManagerUtil::exec_system($command); if ( $high == 0 && $low == 0 ) { $log->info(" ok."); } else { if ( $high == 10 ) { $log->warning("Proceeding."); } else { croak; } } } # Allow write access on master (if read_only==1) $new_master->disable_read_only(); return ( $master_log_file, $master_log_pos ); } sub switch_slaves_internal { my $new_master = shift; my $orig_master_log_file = shift; my $orig_master_log_pos = shift; my $master_log_file = shift; my $master_log_pos = shift; my @alive_slaves = $_server_manager->get_alive_slaves(); $log->info(); $log->info("* Switching slaves in parallel..\n"); $log->info(); my $pm = new Parallel::ForkManager( $#alive_slaves + 1 ); my $wait_fail = 0; my $slave_starting_fail = 0; $pm->run_on_start( sub { my ( $pid, $target ) = @_; $log->info( sprintf( "-- Slave switch on host %s started, pid: %d", $target->get_hostinfo(), $pid ) ); $log->info(); } ); $pm->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; $log->info("Log messages from $target->{hostname} ..."); my $local_file = "$g_workdir/masteronlineswitch_$target->{hostname}_$target->{port}_$start_datetime.log"; $log->info( "\n" . `cat $local_file` ); $log->info("End of log messages from $target->{hostname} ..."); $log->info(); unlink $local_file; if ( $exit_code == 0 ) { $log->info( sprintf( "-- Slave switch on host %s succeeded.", $target->get_hostinfo() ) ); } elsif ( $exit_code == 100 ) { $slave_starting_fail = 1; } else { $wait_fail = 1; $log->info( sprintf( "-- Switching slave on host %s failed, exit code %d", $target->get_hostinfo(), $exit_code ) ); } } ); foreach my $target (@alive_slaves) { # master was already switched next if ( $target->{id} eq $new_master->{id} ); my $pid = $pm->start($target) and next; my $pplog; eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; my $local_file = "$g_workdir/masteronlineswitch_$target->{hostname}_$target->{port}_$start_datetime.log"; unlink $local_file; $pplog = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); $pplog->add( Log::Dispatch::File->new( name => 'file', filename => $local_file, min_level => $target->{log_level}, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append' ) ); $target->current_slave_position($pplog); if ( $target->master_pos_wait( $orig_master_log_file, $orig_master_log_pos, $pplog ) ) { $pm->finish(1); } if ( $_server_manager->change_master_and_start_slave( $target, $new_master, $master_log_file, $master_log_pos, $pplog ) ) { $pm->finish(100); } else { $pm->finish(0); } }; if ($@) { $pplog->error($@) if ($pplog); undef $@; $pm->finish(1); } } $pm->wait_all_children; return ( $wait_fail || $slave_starting_fail ); } sub switch_slaves($$$$$$) { my $orig_master = shift; my $new_master = shift; my $orig_master_log_file = shift; my $orig_master_log_pos = shift; my $master_log_file = shift; my $master_log_pos = shift; my $ret = switch_slaves_internal( $new_master, $orig_master_log_file, $orig_master_log_pos, $master_log_file, $master_log_pos ); $log->info("Unlocking all tables on the orig master:"); $orig_master->unlock_tables(); if ($g_orig_master_is_new_slave) { $log->info("Starting orig master as a new slave.."); if ( exists( $new_master->{use_ip_for_change_master} ) ) { $orig_master->{use_ip_for_change_master} = $new_master->{use_ip_for_change_master}; } if ( $_server_manager->change_master_and_start_slave( $orig_master, $new_master, $master_log_file, $master_log_pos ) ) { $log->error(" Failed!"); $ret = 1; } else { $log->debug(" ok."); } } if ( $ret eq '0' ) { $log->info("All new slave servers switched successfully."); $log->info(); $log->info("* Phase 5: New master cleanup phase.."); $log->info(); if ( $new_master->{skip_reset_slave} ) { $log->info("Skipping RESET SLAVE on the new master."); } else { $ret = $new_master->reset_slave_on_new_master(); } } if ( $ret eq '0' ) { my $message = sprintf( "Switching master to %s completed successfully.", $new_master->get_hostinfo() ); $log->info($message); return 0; } else { my $message = sprintf( "Switching master to %s done, but switching slaves partially failed.", $new_master->get_hostinfo() ); $log->error($message); return 10; } } sub do_master_online_switch { my $error_code = 1; my $orig_master; eval { $orig_master = identify_orig_master(); my $new_master = identify_new_master($orig_master); $log->info("** Phase 1: Configuration Check Phase completed.\n"); $log->info(); if ($g_check_only) { $log->info("--check_only is set. Exit."); $error_code = 0; return; } my ( $orig_master_log_file, $orig_master_log_pos ) = reject_update( $orig_master, $new_master ); $_server_manager->read_slave_status(); my ( $master_log_file, $master_log_pos ) = switch_master( $orig_master, $new_master, $orig_master_log_file, $orig_master_log_pos ); $error_code = switch_slaves( $orig_master, $new_master, $orig_master_log_file, $orig_master_log_pos, $master_log_file, $master_log_pos ); if ( $g_remove_orig_master_conf && !$g_orig_master_is_new_slave && $error_code == 0 ) { MHA::Config::delete_block_and_save( $g_config_file, $orig_master->{id}, $log ); } $_server_manager->release_failover_advisory_lock(); $_server_manager->disconnect_all(); }; if ($@) { if ( $orig_master && $orig_master->{not_error} ) { $log->info($@); } else { MHA::ManagerUtil::print_error( "Got ERROR: $@", $log ); } $_server_manager->disconnect_all() if ($_server_manager); undef $@; } return $error_code; } sub handle_sigint { if ( my $pid = fork ) { waitpid( $pid, 0 ); } elsif ( defined $pid ) { if ($_server_manager) { my @alive_servers = $_server_manager->get_alive_servers(); foreach my $target (@alive_servers) { my $dbh = $target->connect_util(); if ( $dbh && $target->{dbhelper} && $target->{dbhelper}->{connection_id} ) { $log->info( sprintf( "Killing thread %d on %s..", $target->{dbhelper}->{connection_id}, $target->get_hostinfo() ) ); MHA::DBHelper::kill_thread_util( $dbh, $target->{dbhelper}->{connection_id} ); $log->info("ok."); } $dbh->disconnect() if ($dbh); } } exit 0; } exit 1; } sub main { my $sigset = POSIX::SigSet->new(SIGINT); my $sigaction = POSIX::SigAction->new( \&handle_sigint, $sigset, &POSIX::SA_NOCLDSTOP ); POSIX::sigaction( SIGINT, $sigaction ); local @ARGV = @_; my $a = GetOptions( 'global_conf=s' => \$g_global_config_file, 'conf=s' => \$g_config_file, 'check_only' => \$g_check_only, 'new_master_host=s' => \$g_new_master_host, 'new_master_port=i' => \$g_new_master_port, 'workdir=s' => \$g_workdir, 'manager_workdir=s' => \$g_workdir, 'interactive=i' => \$g_interactive, 'orig_master_is_new_slave' => \$g_orig_master_is_new_slave, 'running_updates_limit=i' => \$g_running_updates_limit, 'running_seconds_limit=i' => \$g_running_seconds_limit, 'seconds_behind_master=i' => \$g_seconds_behind_master, 'skip_lock_all_tables' => \$g_skip_lock_all_tables, 'remove_dead_master_conf' => \$g_remove_orig_master_conf, 'remove_orig_master_conf' => \$g_remove_orig_master_conf, 'flush_tables=i' => \$g_flush_tables, ); if ( $#ARGV >= 0 ) { print "Unknown options: "; print $_ . " " foreach (@ARGV); print "\n"; return 1; } unless ($g_config_file) { print "--conf= must be set.\n"; return 1; } my ( $year, $mon, @time ) = reverse( (localtime)[ 0 .. 5 ] ); $start_datetime = sprintf '%04d%02d%02d%02d%02d%02d', $year + 1900, $mon + 1, @time; return do_master_online_switch(); } 1; mha4mysql-manager-0.58/lib/MHA/SSHCheck.pm000066400000000000000000000114121325502577300201110ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::SSHCheck; use strict; use warnings FATAL => 'all'; use English qw(-no_match_vars); use Getopt::Long qw(:config pass_through); use Carp qw(croak); use Time::HiRes qw( sleep ); use Log::Dispatch; use Log::Dispatch::Screen; use Parallel::ForkManager; use MHA::Config; use MHA::ManagerConst; use MHA::ManagerUtil; $| = 1; my $g_global_config_file = $MHA::ManagerConst::DEFAULT_GLOBAL_CONF; my $g_config_file; sub do_ssh_connection_check { my $servers_ref = shift; my $log = shift; my $log_level = shift; my $workdir = shift; my @servers = @$servers_ref; $log->info("Starting SSH connection tests.."); my $failed = 0; my $pm = new Parallel::ForkManager( $#servers + 1 ); $pm->run_on_start( sub { my ( $pid, $target ) = @_; } ); $pm->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; return if ( $target->{skip_init_ssh_check} ); my $local_file = "$workdir/$target->{ssh_host}_$target->{ssh_port}_ssh_check.log"; if ($exit_code) { $failed = 1; if ( -f $local_file ) { $log->error( "\n" . `cat $local_file` ); } } else { if ( -f $local_file ) { $log->debug( "\n" . `cat $local_file` ); } } unlink $local_file; } ); foreach my $src (@servers) { if ( $pm->start($src) ) { # By default, sshd normally accepts only 10 concurrent authentication requests. # If we have lots of alive servers, we might reach this limitation so # shifting child process invocation time a bit to avoid this problem. sleep 0.5; next; } my ( $file, $pplog ); eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; $pm->finish(0) if ( $src->{skip_init_ssh_check} ); $file = "$workdir/$src->{ssh_host}_$src->{ssh_port}_ssh_check.log"; unlink $file; $pplog = Log::Dispatch->new( callbacks => $MHA::ManagerConst::log_fmt ); $pplog->add( Log::Dispatch::File->new( name => 'file', filename => $file, min_level => $log_level, callbacks => $MHA::ManagerConst::add_timestamp, mode => 'append' ) ); foreach my $dst (@servers) { next if ( $dst->{skip_init_ssh_check} ); next if ( $src->{id} eq $dst->{id} ); $pplog->debug( " Connecting via SSH from $src->{ssh_user}\@$src->{ssh_host}($src->{ssh_ip}:$src->{ssh_port}) to $dst->{ssh_user}\@$dst->{ssh_host}($dst->{ssh_ip}:$dst->{ssh_port}).." ); my $command = "ssh $MHA::ManagerConst::SSH_OPT_CHECK -p $src->{ssh_port} $src->{ssh_user}\@$src->{ssh_ip} \"ssh $MHA::ManagerConst::SSH_OPT_CHECK -p $dst->{ssh_port} $dst->{ssh_user}\@$dst->{ssh_ip} exit 0\""; my ( $high, $low ) = MHA::ManagerUtil::exec_system( $command, $file ); if ( $high != 0 || $low != 0 ) { $pplog->error( "SSH connection from $src->{ssh_user}\@$src->{ssh_host}($src->{ssh_ip}:$src->{ssh_port}) to $dst->{ssh_user}\@$dst->{ssh_host}($dst->{ssh_ip}:$dst->{ssh_port}) failed!" ); $pm->finish(1); } $pplog->debug(" ok."); } $pm->finish(0); }; if ($@) { $pplog->error($@) if ($pplog); undef $@; $pm->finish(1); } } $pm->wait_all_children; croak "SSH Configuration Check Failed!\n" if ($failed); $log->info("All SSH connection tests passed successfully."); } sub main { @ARGV = @_; GetOptions( 'global_conf=s' => \$g_global_config_file, 'conf=s' => \$g_config_file, ); unless ($g_config_file) { print "--conf= must be set.\n"; return 1; } my $log = MHA::ManagerUtil::init_log(); my $conf = new MHA::Config( logger => $log, globalfile => $g_global_config_file, file => $g_config_file ); my ( $sc_ref, undef ) = $conf->read_config(); my @servers_config = @$sc_ref; $log = MHA::ManagerUtil::init_log( undef, "debug" ); return do_ssh_connection_check( \@servers_config, $log, "debug", "/tmp" ); } 1; mha4mysql-manager-0.58/lib/MHA/Server.pm000066400000000000000000001006751325502577300177760ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::Server; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use Time::HiRes qw( sleep ); use English qw(-no_match_vars); use MHA::ManagerConst; use MHA::HealthCheck; use MHA::ManagerUtil; use MHA::DBHelper; sub new { my $pkg = shift; bless { logger => undef, }, $pkg; } sub set_logger { my $self = shift; my $logger = shift; $self->{logger} = $logger; } sub version_ge($$) { my $self = shift; my $compare = shift; my $my_version = $self->{mysql_version}; return MHA::NodeUtil::mysql_version_ge( $my_version, $compare ); } sub server_equals { my ( $self, $host, $ip, $port ) = @_; if ( $self->{ip} eq $ip && $self->{port} == $port && $self->{hostname} eq $host ) { return 1; } return 0; } sub get_hostinfo($) { my $self = shift; return "$self->{hostname}($self->{ip}:$self->{port})"; } sub check_slave_status($) { my $self = shift; my $dbhelper = $self->{dbhelper}; return $dbhelper->check_slave_status(); } sub get_failover_advisory_lock { my $self = shift; my $timeout = shift; $timeout = 1 if ( !defined($timeout) ); my $dbh = $self->{dbh}; return MHA::SlaveUtil::get_failover_advisory_lock( $dbh, $timeout ); } sub release_failover_advisory_lock($) { my $self = shift; my $dbh = $self->{dbh}; return MHA::SlaveUtil::release_failover_advisory_lock($dbh); } sub get_monitor_advisory_lock { my $self = shift; my $timeout = shift; $timeout = 1 if ( !defined($timeout) ); my $dbh = $self->{dbh}; return MHA::SlaveUtil::get_monitor_advisory_lock( $dbh, $timeout ); } sub release_monitor_advisory_lock($) { my $self = shift; my $dbh = $self->{dbh}; return MHA::SlaveUtil::release_monitor_advisory_lock($dbh); } sub enable_read_only($) { my $self = shift; my $log = $self->{logger}; my $dbhelper = $self->{dbhelper}; if ( $dbhelper->is_read_only() eq "1" ) { $self->{read_only} = 1; return 0; } else { $log->info( sprintf( "Setting read_only=1 on %s..", $self->get_hostinfo() ) ); if ( $dbhelper->enable_read_only() eq "0" ) { $self->{read_only} = 1; } else { $self->{read_only} = $dbhelper->is_read_only(); } if ( $self->{read_only} eq "1" ) { $log->info(" ok."); return 0; } else { $log->warning(" failed!"); return 1; } } return 1; } sub disable_read_only($) { my $self = shift; my $log = $self->{logger}; my $dbhelper = $self->{dbhelper}; if ( $dbhelper->is_read_only() eq "0" ) { $self->{read_only} = 0; return 0; } else { $log->info( sprintf( "Setting read_only=0 on %s..", $self->get_hostinfo() ) ); if ( $dbhelper->disable_read_only() eq "0" ) { $self->{read_only} = 0; } else { $self->{read_only} = $dbhelper->is_read_only(); } if ( $self->{read_only} eq "0" ) { $log->info(" ok."); return 0; } else { $log->warning(" failed!"); return 1; } } return 1; } sub connect_check { my ( $self, $num_retries, $log, $disconnect ) = @_; my $dbhelper = new MHA::DBHelper(); my $dbh = $dbhelper->connect( $self->{ip}, $self->{port}, $self->{user}, $self->{password}, 0, $num_retries ); if ( !defined($dbh) ) { my $mysql_err = DBI->err; my $msg = sprintf( "Got MySQL error when connecting %s :$mysql_err:", $self->get_hostinfo() ); $msg .= "$DBI::errstr" if ($DBI::errstr); $log->debug($msg) if ($log); if ( $mysql_err && grep ( $_ == $mysql_err, @MHA::ManagerConst::ALIVE_ERROR_CODES ) > 0 ) { $msg .= ", but this is not a MySQL crash. Check MySQL server settings."; if ($log) { $log->error($msg); croak; } else { croak("$msg\n"); } } $self->{dead} = 1; return $MHA::ManagerConst::MYSQL_DEAD_RC; } if ($disconnect) { $dbh->disconnect(); } else { $self->{dbhelper} = $dbhelper; $self->{dbh} = $dbh; } return 0; } sub connect_util { my $self = shift; return MHA::DBHelper::connect_util( $self->{ip}, $self->{port}, $self->{user}, $self->{password} ); } # Failed to connect does not result in script die, because it is sometimes expected. # Configuration error results in script die, because it should not happen if correctly configured. sub connect_and_get_status { my ( $self, $log ) = @_; $log = $self->{logger} unless ($log); if ( $self->connect_check( 5, $log ) == $MHA::ManagerConst::MYSQL_DEAD_RC ) { return; } my $dbhelper = $self->{dbhelper}; my $dbh = $self->{dbh}; $self->{dead} = 0; $log->debug( sprintf( " Connected to: %s, user=%s\n", $self->get_hostinfo(), $self->{user} ) ); $dbhelper->set_long_wait_timeout(); my ( $sstatus, $mip, $mport, $read_only, $relay_purge ) = (); $self->{server_id} = $dbhelper->get_server_id(); $self->{mysql_version} = $dbhelper->get_version(); $self->{has_gtid} = $dbhelper->has_gtid(); $self->{log_bin} = $dbhelper->is_binlog_enabled(); #if log-bin is enabled, check binlog filtering rules on all servers if ( $self->{log_bin} ) { my ( $file, $pos, $binlog_do_db, $binlog_ignore_db ) = $dbhelper->show_master_status(); $self->{File} = $file; $self->{Binlog_Do_DB} = $binlog_do_db; $self->{Binlog_Ignore_DB} = $binlog_ignore_db; } $self->{datadir} = $dbhelper->get_datadir(); $self->{num_slave_workers} = $dbhelper->get_num_workers(); unless ( defined( $self->{num_slave_workers} ) ) { $self->{num_slave_workers} = 0; } $log->debug( sprintf( " Number of slave worker threads on host %s: %d\n", $self->get_hostinfo(), $self->{num_slave_workers} ) ); my %status = $dbhelper->check_slave_status(); $read_only = $dbhelper->is_read_only(); $relay_purge = $dbhelper->is_relay_log_purge(); $sstatus = $status{Status}; if ( $sstatus == 1 ) { # I am not a slave $self->{not_slave} = 1; } elsif ($sstatus) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); croak; } else { $self->{read_only} = $read_only; $self->{relay_purge} = $relay_purge; my $master_ip = MHA::NodeUtil::get_ip( $status{Master_Host} ); unless ($master_ip) { $log->error( sprintf( " Failed to get an IP address of %s! %s replicates from %s, but maybe invalid.", $status{Master_Host}, $self->get_hostinfo(), $status{Master_Host} ) ); croak; } $self->{Master_IP} = $master_ip; $self->{Master_Port} = $status{Master_Port}; $self->{not_slave} = 0; $self->{Master_Host} = $status{Master_Host}; $self->{repl_user} = $status{Master_User} unless ( $self->{repl_user} ); $self->{Auto_Position} = $status{Auto_Position}; $self->{Executed_Gtid_Set} = $status{Executed_Gtid_Set}; # Master_Host is ip address when you use ip address to connect. In this case, you should use ip address to change master. if ( $self->{Master_Host} eq $self->{Master_IP} ) { $self->{use_ip_for_change_master} = 1; } $self->{Replicate_Do_DB} = $status{Replicate_Do_DB}; $self->{Replicate_Ignore_DB} = $status{Replicate_Ignore_DB}; $self->{Replicate_Do_Table} = $status{Replicate_Do_Table}; $self->{Replicate_Ignore_Table} = $status{Replicate_Ignore_Table}; $self->{Replicate_Wild_Do_Table} = $status{Replicate_Wild_Do_Table}; $self->{Replicate_Wild_Ignore_Table} = $status{Replicate_Wild_Ignore_Table}; $self->{relay_log_info_type} = $dbhelper->get_relay_log_info_type( $self->{mysql_version} ); if ( $self->{relay_log_info_type} eq "TABLE" ) { my ( $relay_dir, $current_relay_log ) = MHA::SlaveUtil::get_relay_dir_file_from_table($dbh); $self->{relay_dir} = $relay_dir; $self->{current_relay_log} = $current_relay_log; if ( !$relay_dir || !$current_relay_log ) { $log->error( sprintf( " Getting relay log directory or current relay logfile from replication table failed on %s!", $self->get_hostinfo() ) ); croak; } } else { my $relay_log_info = $dbhelper->get_relay_log_info_path( $self->{mysql_version} ); $self->{relay_log_info} = $relay_log_info; unless ($relay_log_info) { $log->error( sprintf( " Getting relay_log_info failed on %s!", $self->get_hostinfo() ) ); croak; } } } return $self; } sub read_repl_filter { my $self = shift; my $log = $self->{logger}; my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->check_slave_status(1); # show slave status returned empty result. To check replication filtering # rules, temporarily executing CHANGE MASTER to dummy host, and # checking slave status, then resetting slave. if ( $status{Status} == 1 ) { $log->info( sprintf( "%s: SHOW SLAVE STATUS returned empty result. To check replication filtering rules, temporarily executing CHANGE MASTER to a dummy host.", $self->get_hostinfo() ) ); $dbhelper->execute("CHANGE MASTER TO MASTER_HOST='dummy_host'"); %status = $dbhelper->check_slave_status(1); $log->info( sprintf( "%s: Resetting slave pointing to the dummy host.", $self->get_hostinfo() ) ); $self->reset_slave_all(); } if ( $status{Status} == 0 ) { $self->{Replicate_Do_DB} = $status{Replicate_Do_DB}; $self->{Replicate_Ignore_DB} = $status{Replicate_Ignore_DB}; $self->{Replicate_Do_Table} = $status{Replicate_Do_Table}; $self->{Replicate_Ignore_Table} = $status{Replicate_Ignore_Table}; $self->{Replicate_Wild_Do_Table} = $status{Replicate_Wild_Do_Table}; $self->{Replicate_Wild_Ignore_Table} = $status{Replicate_Wild_Ignore_Table}; } } sub check_set_ssh_status { my $self = shift; my $log = shift; my $set_dead = shift; if ( !$self->{dead} ) { if ( MHA::HealthCheck::ssh_check_simple( $self->{ssh_user}, $self->{ssh_host}, $self->{ssh_ip}, $self->{ssh_port}, $self->{logger}, $self->{ssh_connection_timeout} ) || MHA::ManagerUtil::check_node_version_nodie( $log, $self->{ssh_user}, $self->{ssh_host}, $self->{ssh_ip}, $self->{ssh_port} ) ) { $self->{ssh_ok} = 0; $self->{dead} = 1 if ($set_dead); } else { $self->{ssh_ok} = 1; } } } sub check_repl_priv { my ( $self, $log ) = @_; $log = $self->{logger} unless ($log); if ( !$self->{no_master} && $self->{log_bin} && !$self->{not_slave} ) { my $dbhelper = $self->{dbhelper}; unless ( $dbhelper->has_repl_priv( $self->{repl_user} ) ) { $log->error( sprintf( "%s: User %s does not exist or does not have REPLICATION SLAVE privilege! Other slaves can not start replication from this host.", $self->get_hostinfo(), $self->{repl_user} ) ); croak; } } } sub get_and_set_high_max_allowed_packet { my ( $self, $log ) = @_; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; $self->{orig_max_allowed_packet} = $dbhelper->get_max_allowed_packet(); $log->debug( "Current max_allowed_packet is $self->{orig_max_allowed_packet}."); if ( $dbhelper->set_max_allowed_packet_1g() ) { $log->warning("Tentatively setting max_allowed_packet to 1GB failed."); return 1; } else { $log->debug("Tentatively setting max_allowed_packet to 1GB succeeded."); return 0; } } sub set_default_max_allowed_packet { my ( $self, $log ) = @_; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; if ( $dbhelper->set_max_allowed_packet( $self->{orig_max_allowed_packet} ) ) { $log->warning( "Setting max_allowed_packet back to $self->{orig_max_allowed_packet} failed." ); return 1; } else { $log->debug( "Setting max_allowed_packet back to $self->{orig_max_allowed_packet} succeeded." ); return 0; } } sub disable_relay_log_purge { my ( $self, $log ) = @_; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; $dbhelper->disable_relay_log_purge(); $log->debug("Explicitly disabled relay_log_purge."); return 0; } sub current_slave_position { my ( $self, $log ) = @_; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->check_slave_status(); if ( $status{Status} ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); return; } $self->{Master_Log_File} = $status{Master_Log_File}; $self->{Read_Master_Log_Pos} = $status{Read_Master_Log_Pos}; $self->{Relay_Master_Log_File} = $status{Relay_Master_Log_File}; $self->{Exec_Master_Log_Pos} = $status{Exec_Master_Log_Pos}; $self->{Relay_Log_File} = $status{Relay_Log_File}; $self->{Relay_Log_Pos} = $status{Relay_Log_Pos}; $self->{Retrieved_Gtid_Set} = $status{Retrieved_Gtid_Set}; $self->{Executed_Gtid_Set} = $status{Executed_Gtid_Set}; return $self; } #Check whether slave is running and not delayed sub has_replication_problem { my $self = shift; my $allow_delay_seconds = shift; $allow_delay_seconds = 1 unless ($allow_delay_seconds); my $log = $self->{logger}; my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->check_slave_status(); if ( $status{Status} ne '0' ) { $log->error( sprintf( "Getting slave status failed on %s", $self->get_hostinfo() ) ); return 1; } elsif ( $status{Slave_IO_Running} ne "Yes" ) { $log->error( sprintf( "Slave IO thread is not running on %s", $self->get_hostinfo() ) ); return 2; } elsif ( $status{Slave_SQL_Running} ne "Yes" ) { $log->error( sprintf( "Slave SQL thread is not running on %s", $self->get_hostinfo() ) ); return 3; } elsif ( $status{Seconds_Behind_Master} && $status{Seconds_Behind_Master} > $allow_delay_seconds ) { $log->error( sprintf( "Slave is currently behind %d seconds on %s", $status{Seconds_Behind_Master}, $self->get_hostinfo() ) ); return 4; } elsif ( !defined( $status{Seconds_Behind_Master} ) ) { $log->error( sprintf( "Failed to get Seconds_Behind_Master on %s", $self->get_hostinfo() ) ); return 5; } return 0; } sub get_running_threads($$) { my $self = shift; my $mode = shift; my $dbhelper = $self->{dbhelper}; $dbhelper->get_running_threads($mode); } sub get_running_update_threads($$) { my $self = shift; my $mode = shift; my $dbhelper = $self->{dbhelper}; $dbhelper->get_running_update_threads($mode); } sub wait_until_relay_log_applied { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->wait_until_relay_log_applied( $log, $self->{num_slave_workers} ); if ( $status{Status} ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); } return $status{Status}; } sub wait_until_relay_io_log_applied { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->wait_until_relay_io_log_applied( $log, $self->{num_slave_workers} ); if ( $status{Status} ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); } return $status{Status}; } sub get_binlog_position($) { my $self = shift; my $dbhelper = $self->{dbhelper}; return $dbhelper->show_master_status(); } sub check_binlog_stop { my $self = shift; my $log = $self->{logger}; $log->info("Checking binlog writes are stopped or not.."); my ( $file, $pos ) = $self->get_binlog_position(); sleep(0.2); my ( $file2, $pos2 ) = $self->get_binlog_position(); if ( ( $file2 ne $file ) || ( $pos != $pos2 ) ) { $log->error( "Binlog is not stopped! Prev binlog file:pos=$file:$pos, post binlog file:pos=$file2:$pos2." ); return; } else { $log->info(" ok."); } return ( $file, $pos ); } sub disconnect($) { my $self = shift; my $log = $self->{logger}; if ( defined( $self->{dbh} ) ) { $self->{dbh}->disconnect(); $self->{dbh} = undef; $log->debug( sprintf( " Disconnected from %s\n", $self->get_hostinfo() ) ); } else { $log->debug( sprintf( " Already disconnected from %s\n", $self->get_hostinfo() ) ); } } sub reconnect($) { my $self = shift; my $conn_lost = 0; eval { if ( !$self->{dbh}->ping() ) { die; } }; if ($@) { undef $@; $conn_lost = 1; } return $self if ( !$conn_lost ); my $dbh = $self->{dbhelper} ->connect( $self->{ip}, $self->{port}, $self->{user}, $self->{password} ); if ( !defined($dbh) ) { $self->{dead} = 1; return; } $self->{dbhelper}->set_long_wait_timeout(); $self->{dbh} = $dbh; return $self; } sub flush_tables($) { my $self = shift; my $dbhelper = $self->{dbhelper}; my $log = $self->{logger}; my $errstr; $log->info( "Executing FLUSH NO_WRITE_TO_BINLOG TABLES. This may take long time.."); if ( $errstr = $dbhelper->flush_tables_nolog() ) { $log->error( " Failed! " . $errstr ); } else { $log->info(" ok."); } } sub lock_all_tables($) { my $self = shift; my $dbhelper = $self->{dbhelper}; my $log = $self->{logger}; my $errstr; $log->info("Executing FLUSH TABLES WITH READ LOCK.."); if ( $errstr = $dbhelper->flush_tables_with_read_lock() ) { $log->error( " Failed! " . $errstr ); return 1; } else { $log->info(" ok."); } return 0; } sub unlock_tables($) { my $self = shift; my $dbhelper = $self->{dbhelper}; my $log = $self->{logger}; my $errstr; $log->info("Executing UNLOCK TABLES.."); if ( $errstr = $dbhelper->unlock_tables() ) { $log->error( "Failed! " . $errstr ); return 1; } else { $log->info(" ok."); } return 0; } sub reset_slave_all { my $self = shift; my $dbhelper = $self->{dbhelper}; if ( !$self->version_ge("5.5.0") ) { $dbhelper->reset_slave_by_change_master(); } $dbhelper->reset_slave_master_host(); } # Let the server to return nothing at SHOW SLAVE STATUS (Without this, the new master still points to the previous master) sub reset_slave_info($) { my $self = shift; my $log = $self->{logger}; my $dbhelper = $self->{dbhelper}; $log->debug(" Clearing slave info.."); if ( $self->stop_slave() ) { $log->error(" Stopping slave failed!"); return 1; } $self->reset_slave_all(); my %status = $dbhelper->check_slave_status(); if ( $status{Status} == 1 ) { $log->debug( " SHOW SLAVE STATUS shows new master does not replicate from anywhere. OK." ); return 0; } else { $log->error( " SHOW SLAVE STATUS shows new master replicates from somewhere. Check for details!" ); return 1; } } sub reset_slave_on_new_master($) { my $self = shift; my $dbhelper = $self->{dbhelper}; my $log = $self->{logger}; my $ret = $self->reset_slave_info(); if ($ret) { my $message = " $self->{hostname}: Resetting slave info failed."; $log->error($message); return 1; } else { my $message = " $self->{hostname}: Resetting slave info succeeded."; $log->info($message); return 0; } } # It is possible that slave io thread has not started or established yet # when you execute "START SLAVE". It should start within 0-4 seconds. # So we wait some time until slave starts. # Return: 0: OK 1: NG sub wait_until_slave_starts($$) { my $self = shift; my $type = shift; my $log = $self->{logger}; my $dbhelper = $self->{dbhelper}; my $retry_count = 100; my $interval = 0.1; if ( $self->{has_gtid} ) { $interval = 1; $retry_count = 600; } for ( my $i = 0 ; $i < $retry_count ; $i++ ) { my %status = $dbhelper->check_slave_status(); if ( $status{Status} ) { # on GTID mode, it may take time to have Master_Log_File if ( !$self->{has_gtid} || ( $self->{has_gtid} && $status{Errstr} ) ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); return 1; } } if ( $type eq "IO" ) { return 0 if ( $status{Slave_IO_Running} eq "Yes" ); } elsif ( $type eq "SQL" ) { return 0 if ( $status{Slave_SQL_Running} eq "Yes" ); } else { return 0 if ( $status{Slave_IO_Running} eq "Yes" && $status{Slave_SQL_Running} eq "Yes" ); } if ( $status{Slave_SQL_Running} eq "No" && $status{Last_Errno} ne '0' ) { $log->error( sprintf( "SQL Thread could not be started on %s! Check slave status.", $self->get_hostinfo() ) ); $log->error( sprintf( " Last Error= %d, Last Error=%s", $status{Last_Errno}, $status{Last_Error} ) ); return 1; } sleep($interval); } $log->error( sprintf( "Slave could not be started on %s! Check slave status.", $self->get_hostinfo() ) ); return 1; } sub wait_until_slave_stops { my $self = shift; my $type = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my $retry_count = 100; my $interval = 0.1; if ( $self->{has_gtid} ) { $interval = 1; $retry_count = 600; } for ( my $i = 0 ; $i < $retry_count ; $i++ ) { my %status = $dbhelper->check_slave_status(); if ( $status{Status} ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); return 1; } if ( $type eq "IO" ) { return 0 if ( $status{Slave_IO_Running} eq "No" ); } elsif ( $type eq "SQL" ) { return 0 if ( $status{Slave_SQL_Running} eq "No" ); } else { return 0 if ( $status{Slave_IO_Running} eq "No" && $status{Slave_SQL_Running} eq "No" ); } sleep($interval); } $log->error( sprintf( "Slave could not be stopped on %s! Check slave status.", $self->get_hostinfo() ) ); return 1; } sub stop_slave { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my ($sstatus) = (); $log->debug( sprintf( " Stopping slave IO/SQL thread on %s..", $self->get_hostinfo() ) ); $dbhelper->stop_slave(); if ( $self->wait_until_slave_stops( 'ALL', $log ) ) { $log->error( sprintf( "Stopping slave IO/SQL thread on %s Failed!", $self->get_hostinfo() ) ); return 1; } $log->debug(" done."); return 0; } sub start_slave { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my ($sstatus) = (); $log->debug( sprintf( " Starting slave IO/SQL thread on %s..", $self->get_hostinfo() ) ); $dbhelper->start_slave(); if ( $self->wait_until_slave_starts( 'ALL', $log ) ) { $log->error( sprintf( "Starting slave IO/SQL thread on %s failed!", $self->get_hostinfo() ) ); return 1; } $log->debug(" done."); return 0; } sub stop_io_thread { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my ($sstatus) = (); $log->debug( sprintf( " Stopping IO thread on %s..", $self->get_hostinfo() ) ); $dbhelper->stop_io_thread(); if ( $self->wait_until_slave_stops( 'IO', $log ) ) { $log->error( sprintf( "Failed to stop IO thread on %s!", $self->get_hostinfo() ) ); return 1; } $log->debug( sprintf( " Stop IO thread on %s done.", $self->get_hostinfo() ) ); return 0; } sub stop_sql_thread { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my ($sstatus) = (); $log->debug( sprintf( " Stopping SQL thread on %s..", $self->get_hostinfo() ) ); $dbhelper->stop_sql_thread(); if ( $self->wait_until_slave_stops( 'SQL', $log ) ) { $log->error( sprintf( "Stopping SQL thread on %s failed!", $self->get_hostinfo() ) ); return 1; } $log->debug(" done."); return 0; } sub is_sql_thread_error { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->check_slave_status(); if ( $status{Status} ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); return 1; } return 0 if ( $status{Slave_SQL_Running} eq "Yes" ); if ( $status{Slave_SQL_Running} eq "No" && $status{Last_Errno} eq '0' ) { $log->warning( sprintf( "SQL Thread is stopped(no error) on %s", $self->get_hostinfo() ) ); return 0; } $log->error( sprintf( "SQL Thread is stopped(error) on %s! Errno:%s, Error:%s", $self->get_hostinfo(), $status{Last_Errno}, $status{Last_Error} ) ); return 1; } sub start_sql_thread_if { my $self = shift; my $log = shift; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my %status = $dbhelper->check_slave_status(); if ( $status{Status} ) { my $msg = sprintf( "Checking slave status failed on %s.", $self->get_hostinfo() ); $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); return 1; } return 0 if ( $status{Slave_SQL_Running} eq "Yes" ); $log->info( sprintf( " Starting SQL thread on %s ..", $self->get_hostinfo() ) ); $dbhelper->start_sql_thread(); if ( $self->wait_until_slave_starts( 'SQL', $log ) ) { $log->info(" Failed!"); return 1; } $log->info(" done."); return 0; } sub gtid_wait { my ( $self, $exec_gtid, $log ) = @_; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my $res = $dbhelper->gtid_wait($exec_gtid); if ( !defined($res) ) { $log->error( sprintf( "gtid_wait(%s) returned NULL on %s. Maybe SQL thread was aborted?", $exec_gtid, $self->get_hostinfo() ) ); return 1; } if ( $res >= 0 ) { $log->info( sprintf( " gtid_wait(%s) completed on %s. Executed %d events.", $exec_gtid, $self->get_hostinfo(), $res ) ); return 0; } else { $log->error( sprintf( "gtid_wait(%s) got error on %s: $res", $exec_gtid, $self->get_hostinfo() ) ); return 1; } } sub master_pos_wait_internal { my ( $self, $binlog_file, $binlog_pos, $log ) = @_; $log = $self->{logger} unless ($log); my $dbhelper = $self->{dbhelper}; my $res = $dbhelper->master_pos_wait( $binlog_file, $binlog_pos ); if ( !defined($res) ) { $log->error( sprintf( "master_pos_wait(%s:%d) returned NULL on %s. Maybe SQL thread was aborted?", $binlog_file, $binlog_pos, $self->get_hostinfo() ) ); return 1; } if ( $res >= 0 ) { $log->info( sprintf( " master_pos_wait(%s:%d) completed on %s. Executed %d events.", $binlog_file, $binlog_pos, $self->get_hostinfo(), $res ) ); return 0; } else { $log->error( sprintf( "master_pos_wait(%s:%d) got error on %s: $res", $binlog_file, $binlog_pos, $self->get_hostinfo() ) ); return 1; } } # We do not reset slave here sub master_pos_wait { my ( $self, $binlog_file, $binlog_pos, $log ) = @_; $log = $self->{logger} unless ($log); $log->info( sprintf( " Waiting to execute all relay logs on %s..", $self->get_hostinfo() ) ); my $ret = $self->master_pos_wait_internal( $binlog_file, $binlog_pos, $log ); if ($ret) { return $ret; } $log->info(" done."); $self->stop_sql_thread($log); return 0; } sub print_server { my $self = shift; my $log = $self->{logger}; my $ssh_str = ""; $ssh_str = " Not reachable via SSH" if ( defined( $self->{ssh_ok} ) && $self->{ssh_ok} == 0 ); my $version_str = ""; $version_str = " Version=$self->{mysql_version}" if ( $self->{mysql_version} ); $version_str = $version_str . " (oldest major version between slaves)" if ( defined( $self->{oldest_major_version} ) && $self->{oldest_major_version} >= 1 ); my $binlog_str = ""; if ( defined( $self->{log_bin} ) ) { if ( $self->{log_bin} > 0 ) { $binlog_str = " log-bin:enabled"; } else { $binlog_str = " log-bin:disabled"; } } $log->info( " " . $self->get_hostinfo() . $ssh_str . $version_str . $binlog_str ); if ( $self->{has_gtid} ) { $log->info(" GTID ON"); } $log->debug(" Relay log info repository: $self->{relay_log_info_type}") if ( $self->{relay_log_info_type} ); if ( $self->{Master_IP} && $self->{Master_Port} ) { $log->info( sprintf( " Replicating from %s(%s:%d)", $self->{Master_Host}, $self->{Master_IP}, $self->{Master_Port} ) ); if ( $self->{no_master} ) { $log->info(" Not candidate for the new Master (no_master is set)"); } elsif ( $self->{candidate_master} ) { $log->info( " Primary candidate for the new Master (candidate_master is set)"); } } } sub print_filter { my $self = shift; my $is_master = shift; my $print_repl = shift; $is_master = 0 unless ($is_master); $print_repl = 1 if ( !defined($print_repl) ); my $str = ""; $str .= "$self->{hostname}"; $str .= " (current_master)" if ($is_master); $str .= " ($self->{node_label})" if ( $self->{node_label} ); $str .= "\n"; $str .= sprintf( " Binlog_Do_DB: %s\n", $self->{Binlog_Do_DB} ? $self->{Binlog_Do_DB} : "" ); $str .= sprintf( " Binlog_Ignore_DB: %s\n", $self->{Binlog_Ignore_DB} ? $self->{Binlog_Ignore_DB} : "" ); if ($print_repl) { $str .= sprintf( " Replicate_Do_DB: %s\n", $self->{Replicate_Do_DB} ? $self->{Replicate_Do_DB} : "" ); $str .= sprintf( " Replicate_Ignore_DB: %s\n", $self->{Replicate_Ignore_DB} ? $self->{Replicate_Ignore_DB} : "" ); $str .= sprintf( " Replicate_Do_Table: %s\n", $self->{Replicate_Do_Table} ? $self->{Replicate_Do_Table} : "" ); $str .= sprintf( " Replicate_Ignore_Table: %s\n", $self->{Replicate_Ignore_Table} ? $self->{Replicate_Ignore_Table} : "" ); $str .= sprintf( " Replicate_Wild_Do_Table: %s\n", $self->{Replicate_Wild_Do_Table} ? $self->{Replicate_Wild_Do_Table} : "" ); $str .= sprintf( " Replicate_Wild_Ignore_Table: %s\n", $self->{Replicate_Wild_Ignore_Table} ? $self->{Replicate_Wild_Ignore_Table} : "" ); } $str .= "\n"; return $str; } sub get_ssh_args_if { my $self = shift; my $arg_number = shift; my $type = shift; my $ssh_reachable = shift; my $arg = " "; my $head = ""; if ( $type eq "orig" ) { $head = " --orig_master_"; } elsif ( $type eq "new" ) { $head = " --new_master_"; } elsif ( $type eq "shutdown" ) { $head = " --"; } if ( $self->{hostname} ne $self->{ssh_host} || $self->{ip} ne $self->{ssh_ip} ) { $arg .= $head . "ssh_host=$self->{ssh_host}" . $head . "ssh_ip=$self->{ssh_ip}"; } if ( $self->{ssh_port} ne 22 ) { $arg .= $head . "ssh_port=$self->{ssh_port}"; } if ( $MHA::ManagerConst::USE_SSH_OPTIONS && $arg_number <= 1 ) { if ($ssh_reachable) { $arg .= " --ssh_options='$MHA::NodeConst::SSH_OPT_ALIVE' "; } else { $arg .= " --ssh_options='$MHA::ManagerConst::SSH_OPT_CHECK' "; } } return $arg; } 1; mha4mysql-manager-0.58/lib/MHA/ServerManager.pm000066400000000000000000001264441325502577300212730ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA package MHA::ServerManager; use strict; use warnings FATAL => 'all'; use Carp qw(croak); use English qw(-no_match_vars); use MHA::SlaveUtil; use MHA::DBHelper; use MHA::Server; use MHA::ManagerConst; use Parallel::ForkManager; sub new { my $class = shift; my $self = { servers => [], dead_servers => [], alive_servers => [], alive_slaves => [], failed_slaves => [], latest_slaves => [], oldest_slaves => [], unmanaged_slaves => [], orig_master => undef, new_master => undef, logger => undef, @_, }; return bless $self, $class; } sub set_servers($$) { my $self = shift; my $servers_ref = shift; $self->{servers} = $servers_ref; } sub set_latest_slaves($$) { my $self = shift; my $servers_ref = shift; $self->{latest_slaves} = $servers_ref; } sub set_oldest_slaves($$) { my $self = shift; my $servers_ref = shift; $self->{oldest_slaves} = $servers_ref; } sub set_unmanaged_slaves($$) { my $self = shift; my $servers_ref = shift; $self->{unmanaged_slaves} = $servers_ref; } sub get_servers($) { my $self = shift; return @{ $self->{servers} }; } sub get_dead_servers($) { my $self = shift; return @{ $self->{dead_servers} }; } sub get_alive_servers($) { my $self = shift; return @{ $self->{alive_servers} }; } sub get_alive_slaves($) { my $self = shift; return @{ $self->{alive_slaves} }; } sub get_failed_slaves($) { my $self = shift; return @{ $self->{failed_slaves} }; } sub get_latest_slaves($) { my $self = shift; return @{ $self->{latest_slaves} }; } sub get_oldest_slaves($) { my $self = shift; return @{ $self->{oldest_slaves} }; } sub get_unmanaged_slaves($) { my $self = shift; return @{ $self->{unmanaged_slaves} }; } sub add_dead_server($$) { my $self = shift; my $server = shift; push @{ $self->{dead_servers} }, $server; } sub add_alive_server($$) { my $self = shift; my $server = shift; push @{ $self->{alive_servers} }, $server; } sub add_alive_slave($$) { my $self = shift; my $server = shift; push @{ $self->{alive_slaves} }, $server; } sub add_failed_slave($$) { my $self = shift; my $server = shift; push @{ $self->{failed_slaves} }, $server; } sub add_unmanaged_slave($$) { my $self = shift; my $server = shift; push @{ $self->{unmanaged_slaves} }, $server; } sub set_orig_master($$) { my $self = shift; my $server = shift; $self->{orig_master} = $server; $server->{orig_master} = 1; } sub get_orig_master($) { my $self = shift; return $self->{orig_master}; } sub init_servers($) { my $self = shift; my $log = $self->{logger}; my @servers = $self->get_servers(); $self->{dead_servers} = []; $self->{alive_servers} = []; $self->{alive_slaves} = []; $self->{failed_slaves} = []; $self->{unmanaged_slaves} = []; foreach my $server (@servers) { if ( $server->{dead} ) { $self->add_dead_server($server); } elsif ( $server->{unmanaged} ) { $self->add_unmanaged_slave($server); } else { $self->add_alive_server($server); if ( $server->{not_slave} eq '0' && !$server->{orig_master} ) { if ( !$server->is_sql_thread_error() && !$server->{lack_relay_log} ) { $self->add_alive_slave($server); } else { $self->add_failed_slave($server); } } } } my @alive_servers = $self->get_alive_servers(); if ( $#alive_servers <= -1 ) { $log->error("There is no alive server. We can't do failover"); croak; } my @alive_slaves = $self->get_alive_slaves(); if ( $#alive_slaves <= -1 ) { $log->error("There is no alive slave. We can't do failover"); croak; } } sub init_binlog_server { my $binlog_server_ref = shift; my $log = shift; my @binlog_servers = @$binlog_server_ref; my $num_alive_binlog_servers = 0; foreach my $server (@binlog_servers) { unless ( $server->{logger} ) { $server->{logger} = $log; } if ( MHA::HealthCheck::ssh_check_simple( $server->{ssh_user}, $server->{ssh_host}, $server->{ssh_ip}, $server->{ssh_port}, $server->{logger}, $server->{ssh_connection_timeout} ) ) { $log->warning("Failed to SSH to binlog server $server->{hostname}"); $server->{ssh_reachable} = 0; } else { if ( MHA::ManagerUtil::get_node_version( $server->{logger}, $server->{ssh_user}, $server->{ssh_host}, $server->{ssh_ip}, $server->{ssh_port} ) ) { $log->info("Binlog server $server->{hostname} is reachable."); $server->{ssh_reachable} = 1; $num_alive_binlog_servers++; } else { $log->warning( "Failed to get MHA Node version from binlog server $server->{hostname}" ); $server->{ssh_reachable} = 0; } } } if ( $#binlog_servers >= 0 && $num_alive_binlog_servers <= 0 ) { $log->error("Binlog Server is defined but there is no alive server."); croak; } } sub set_logger($$) { my $self = shift; my $logger = shift; $self->{logger} = $logger; } sub connect_all_and_read_server_status($$$$) { my $self = shift; my $dead_master_host = shift; my $dead_master_ip = shift; my $dead_master_port = shift; my $log = $self->{logger}; my @servers = $self->get_servers(); $log->debug("Connecting to servers.."); my $should_die = 0; my $connection_checker = new Parallel::ForkManager( $#servers + 1 ); $connection_checker->run_on_start( sub { my ( $pid, $target ) = @_; } ); $connection_checker->run_on_finish( sub { my ( $pid, $exit_code, $target ) = @_; if ( $exit_code == $MHA::ManagerConst::MYSQL_DEAD_RC ) { $target->{dead} = 1; } elsif ($exit_code) { $should_die = 1; } } ); foreach my $target (@servers) { unless ( $target->{logger} ) { $target->{logger} = $log; } $connection_checker->start($target) and next; eval { $SIG{INT} = $SIG{HUP} = $SIG{QUIT} = $SIG{TERM} = "DEFAULT"; if ( $dead_master_host && $dead_master_ip && $dead_master_port ) { if ( $target->server_equals( $dead_master_host, $dead_master_ip, $dead_master_port ) ) { $connection_checker->finish($MHA::ManagerConst::MYSQL_DEAD_RC); } } my $rc = $target->connect_check( 2, $log, 1 ); $connection_checker->finish($rc); }; if ($@) { $log->error($@); undef $@; $connection_checker->finish(1); } $connection_checker->finish(0); } $connection_checker->wait_all_children; if ($should_die) { $log->error("Got fatal error, stopping operations"); croak; } foreach my $target (@servers) { next if ( $target->{dead} ); $target->connect_and_get_status(); } $self->init_servers(); $self->compare_slave_version(); $log->debug("Connecting to servers done."); $self->validate_current_master(); $self->{gtid_failover_mode} = $self->get_gtid_status(); $log->info( sprintf( "GTID failover mode = %d", $self->{gtid_failover_mode} ) ); } sub get_oldest_version($) { my $self = shift; my @servers = $self->get_alive_servers(); my $oldest_version; foreach my $server (@servers) { if ( $server->{oldest_major_version} ) { $oldest_version = $server->{mysql_version}; last; } } return $oldest_version; } sub compare_slave_version($) { my $self = shift; my @servers = $self->get_alive_servers(); my $log = $self->{logger}; $log->debug(" Comparing MySQL versions.."); my $min_major_version; foreach (@servers) { my $dbhelper = $_->{dbhelper}; next if ( $_->{dead} || $_->{not_slave} ); my $parsed_major_version = MHA::NodeUtil::parse_mysql_major_version( $_->{mysql_version} ); if (!$min_major_version || $parsed_major_version < $min_major_version ) { $min_major_version = $parsed_major_version; } } foreach (@servers) { my $dbhelper = $_->{dbhelper}; next if ( $_->{dead} || $_->{not_slave} ); my $parsed_major_version = MHA::NodeUtil::parse_mysql_major_version( $_->{mysql_version} ); if ( $min_major_version == $parsed_major_version ) { $_->{oldest_major_version} = 1; } else { $_->{oldest_major_version} = 0; } } $log->debug(" Comparing MySQL versions done."); } sub print_filter_rules($$) { my $self = shift; my $master = shift; my $log = $self->{logger}; my $msg = "Bad Binlog/Replication filtering rules:\n"; $msg .= $master->print_filter(1) if ( $master && !$master->{dead} ); my @slaves = $self->get_alive_slaves(); foreach my $slave (@slaves) { $msg .= $slave->print_filter(); } $log->warning($msg); } sub validate_repl_filter($$) { my $self = shift; my $master = shift; my $log = $self->{logger}; $log->info("Checking replication filtering settings.."); my $binlog_do_db; my $binlog_ignore_db; # If master is alive if ( $master && !$master->{dead} ) { $binlog_do_db = $master->{Binlog_Do_DB}; $binlog_ignore_db = $master->{Binlog_Ignore_DB}; $log->info( " binlog_do_db= $binlog_do_db, binlog_ignore_db= $binlog_ignore_db"); } my @slaves = $self->get_alive_slaves(); my $replicate_do_db; my $replicate_ignore_db; my $replicate_do_table; my $replicate_ignore_table; my $replicate_wild_do_table; my $replicate_wild_ignore_table; foreach (@slaves) { $replicate_do_db = $_->{Replicate_Do_DB} unless ($replicate_do_db); $replicate_ignore_db = $_->{Replicate_Ignore_DB} unless ($replicate_ignore_db); $replicate_do_table = $_->{Replicate_Do_Table} unless ($replicate_do_table); $replicate_ignore_table = $_->{Replicate_Ignore_Table} unless ($replicate_ignore_table); $replicate_wild_do_table = $_->{Replicate_Wild_Do_Table} unless ($replicate_wild_do_table); $replicate_wild_ignore_table = $_->{Replicate_Wild_Ignore_Table} unless ($replicate_wild_ignore_table); if ( $_->{log_bin} ) { $binlog_do_db = $_->{Binlog_Do_DB} unless ($binlog_do_db); $binlog_ignore_db = $_->{Binlog_Ignore_DB} unless ($binlog_ignore_db); } if ( $replicate_do_db ne $_->{Replicate_Do_DB} || $replicate_ignore_db ne $_->{Replicate_Ignore_DB} || $replicate_do_table ne $_->{Replicate_Do_Table} || $replicate_ignore_table ne $_->{Replicate_Ignore_Table} || $replicate_wild_do_table ne $_->{Replicate_Wild_Do_Table} || $replicate_wild_ignore_table ne $_->{Replicate_Wild_Ignore_Table} ) { $log->error( sprintf( "Replication filtering check failed on %s! All slaves must have same replication filtering rules. Check SHOW SLAVE STATUS output and set my.cnf correctly.", $_->get_hostinfo() ) ); $self->print_filter_rules($master); return 1; } if ( $_->{log_bin} ) { if ( $binlog_do_db ne $_->{Binlog_Do_DB} || $binlog_ignore_db ne $_->{Binlog_Ignore_DB} ) { $log->error( sprintf( "Binlog filtering check failed on %s! All log-bin enabled servers must have same binlog filtering rules (same binlog-do-db and binlog-ignore-db). Check SHOW MASTER STATUS output and set my.cnf correctly.", $_->get_hostinfo() ) ); $self->print_filter_rules($master); return 1; } } } if ( $binlog_do_db && $replicate_do_db ) { if ( $binlog_do_db ne $replicate_do_db ) { $log->error( sprintf( "binlog_do_db on master(%s) must be the same as replicate_do_db on slaves(%s).", $binlog_do_db, $replicate_do_db ) ); $self->print_filter_rules($master); return 1; } } if ( $binlog_ignore_db && $replicate_ignore_db ) { if ( $binlog_ignore_db ne $replicate_ignore_db ) { $log->error( sprintf( "binlog_ignore_db on master(%s) must be the same as replicate_ignore_db on slaves(%s).", $binlog_ignore_db, $replicate_ignore_db ) ); $self->print_filter_rules($master); return 1; } } $log->info(" Replication filtering check ok."); return 0; } sub validate_num_alive_servers($$$) { my $self = shift; my $current_master = shift; my $ignore_fail_check = shift; my $log = $self->{logger}; my @dead_servers = $self->get_dead_servers(); my @failed_slaves = $self->get_failed_slaves(); foreach (@dead_servers) { next if ( $_->{id} eq $current_master->{id} ); next if ( $ignore_fail_check && $_->{ignore_fail} ); $log->error( sprintf( " Server %s is dead, but must be alive! Check server settings.", $_->get_hostinfo() ) ); croak; } foreach (@failed_slaves) { next if ( $ignore_fail_check && $_->{ignore_fail} ); $log->error( sprintf( " Replication on %s fails! Check server settings.", $_->get_hostinfo() ) ); croak; } return 0; } # Check the following # 1. All slaves are read_only (INFO) # 2. All slaves see the same master ip/port (ERROR) # 3. All slaves set relay_log_purge=0 (WARN) # 4. All slaves have same replication filter rules with a master (ERROR) # return 0: ok, others: NG sub validate_slaves($$$) { my $self = shift; my $check_repl_filter = shift; my $master = shift; my $log = $self->{logger}; my @slaves = $self->get_alive_slaves(); my ( $mip, $mport ) = (); my $error = 0; $log->info("Checking slave configurations.."); foreach (@slaves) { if ( $_->{read_only} ne '1' ) { $log->info( sprintf( " read_only=1 is not set on slave %s.\n", $_->get_hostinfo() ) ); } if ( $_->{relay_purge} ne '0' && !$_->{has_gtid} ) { $log->warning( sprintf( " relay_log_purge=0 is not set on slave %s.\n", $_->get_hostinfo() ) ); } if ( $_->{log_bin} eq '0' ) { $log->warning( sprintf( " log-bin is not set on slave %s. This host cannot be a master.\n", $_->get_hostinfo() ) ); } } $error = $self->validate_repl_filter($master) if ($check_repl_filter); return $error; } sub get_alive_server_by_ipport { my $self = shift; my $ip = shift; my $port = shift; $self->get_server_by_ipport( $ip, $port, 1 ); } sub get_server_by_ipport { my $self = shift; my $ip = shift; my $port = shift; my $alive_only = shift; my @servers; if ($alive_only) { @servers = $self->get_alive_servers(); } else { @servers = $self->get_servers(); } foreach (@servers) { if ( $_->{ip} eq $ip && $_->{port} == $port ) { return $_; } } return; } sub get_alive_server_by_hostport { my $self = shift; my $host = shift; my $port = shift; my @servers = $self->get_alive_servers(); foreach (@servers) { if ( $_->{hostname} eq $host && $_->{port} == $port ) { return $_; } } return; } sub get_server_from_by_id { my $self = shift; my $servers_ref = shift; my $id = shift; my @servers = @$servers_ref; foreach (@servers) { if ( $_->{id} eq $id ) { return $_; } } return; } sub get_alive_server_by_id { my $self = shift; my $id = shift; my @alive_servers = $self->get_alive_servers(); foreach (@alive_servers) { if ( $_->{id} eq $id ) { return $_; } } return; } sub get_alive_slave_by_id { my $self = shift; my $id = shift; my @alive_slaves = $self->get_alive_slaves(); foreach (@alive_slaves) { if ( $_->{id} eq $id ) { return $_; } } return; } sub get_master_by_slave { my $self = shift; my $slave = shift; return $self->get_server_by_ipport( $slave->{Master_IP}, $slave->{Master_Port} ); } sub validate_current_master($) { my $self = shift; my $log = $self->{logger}; my @alive_servers = $self->get_alive_servers(); my %master_hash; my $num_slaves = 0; my $not_slave_servers = 0; foreach (@alive_servers) { if ( $_->{not_slave} eq '0' ) { $master_hash{"$_->{Master_IP}:$_->{Master_Port}"} = $_; $num_slaves++; } else { $not_slave_servers++; } } if ( $not_slave_servers >= 2 ) { $log->error( "There are $not_slave_servers non-slave servers! MHA manages at most one non-slave server. Check configurations." ); croak; } if ( $num_slaves < 1 ) { $log->error( "There is not any alive slave! Check slave settings for details."); croak; } # verify masters exist in a config file my $master; foreach my $key ( keys(%master_hash) ) { my $slave = $master_hash{$key}; $master = $self->get_master_by_slave($slave); unless ($master) { $log->error( sprintf( "Master %s:%d from which slave %s replicates is not defined in the configuration file!", $slave->{Master_IP}, $slave->{Master_Port}, $slave->get_hostinfo() ) ); croak; } } my $real_master; if ( keys(%master_hash) >= 2 ) { $real_master = $self->get_primary_master( \%master_hash ); } else { $real_master = $master; $self->set_orig_master($real_master); } $self->validate_master_ip_port($real_master); return $real_master; } sub validate_master_ip_port { my $self = shift; my $real_master = shift; my $log = $self->{logger}; my $has_unmanaged_slaves = 0; my @alive_servers = $self->get_alive_servers(); foreach my $slave (@alive_servers) { next if ( $slave->{id} eq $real_master->{id} ); unless ( $self->get_alive_slave_by_id( $slave->{id} ) ) { $log->error( sprintf( "Server %s is alive, but does not work as a slave!", $slave->get_hostinfo() ) ); croak; } if ( !( ( $slave->{Master_IP} eq $real_master->{ip} ) && ( $slave->{Master_Port} == $real_master->{port} ) ) ) { if ( $slave->{multi_tier_slave} ) { $slave->{unmanaged} = 1; $has_unmanaged_slaves = 1; } else { my $msg = sprintf( "Slave %s replicates from %s:%d, but real master is %s!", $slave->get_hostinfo(), $slave->{Master_Host}, $slave->{Master_Port}, $real_master->get_hostinfo() ); $log->error($msg); croak; } } } if ($has_unmanaged_slaves) { $self->init_servers(); } } sub get_multi_master_print_info { my $self = shift; my $master_hash_ref = shift; my %master_hash = %$master_hash_ref; my $str = ""; foreach my $key ( keys(%master_hash) ) { my $slave = $master_hash{$key}; my $master = $self->get_master_by_slave($slave); $str .= "Master " . $master->get_hostinfo(); $str .= ", replicating from $master->{Master_Host}($master->{Master_IP}:$master->{Master_Port})" if ( $master->{Master_Host} ); $str .= ", read-only" if ( $master->{read_only} ); $str .= ", dead" if ( $master->{dead} ); $str .= "\n"; } $str .= "\n"; return $str; } sub get_primary_master { my $self = shift; my $master_hash_ref = shift; my $log = $self->{logger}; my @alive_servers = $self->get_alive_servers(); my %master_hash = %$master_hash_ref; my $num_real_masters = 0; my $real_master; foreach my $key ( keys(%master_hash) ) { my $slave = $master_hash{$key}; my $master = $self->get_master_by_slave($slave); next if ( !$master->{dead} && $master->{read_only} ); $real_master = $master; $num_real_masters++; } if ( $num_real_masters < 1 ) { $log->error( sprintf( "Multi-master configuration is detected, but all of them are read-only! Check configurations for details. Master configurations are as below: \n%s", $self->get_multi_master_print_info($master_hash_ref) ) ); croak; } elsif ( $num_real_masters >= 2 ) { $log->error( sprintf( "Multi-master configuration is detected, but two or more masters are either writable (read-only is not set) or dead! Check configurations for details. Master configurations are as below: \n%s", $self->get_multi_master_print_info($master_hash_ref) ) ); croak; } else { $self->set_orig_master($real_master); $log->info( sprintf( "Multi-master configuration is detected. Current primary(writable) master is %s", $real_master->get_hostinfo() ) ); $log->info( sprintf( "Master configurations are as below: \n%s", $self->get_multi_master_print_info($master_hash_ref) ) ); $self->init_servers(); } return $real_master; } sub get_candidate_masters($) { my $self = shift; my $log = $self->{logger}; my @servers = $self->get_servers(); my @ret_servers = (); foreach (@servers) { next if ( $_->{dead} eq '1' ); if ( $_->{candidate_master} >= 1 ) { push( @ret_servers, $_ ); } } return @ret_servers; } sub print_dead_servers { my $self = shift; $self->print_servers( $self->{dead_servers} ); } sub print_alive_servers { my $self = shift; my $log = $self->{logger}; my @alive_servers = $self->get_alive_servers(); foreach (@alive_servers) { $log->info( " " . $_->get_hostinfo() ); } } sub print_alive_slaves { my $self = shift; $self->print_servers( $self->{alive_slaves} ); } sub print_latest_slaves { my $self = shift; $self->print_servers( $self->{latest_slaves} ); } sub print_oldest_slaves { my $self = shift; $self->print_servers( $self->{oldest_slaves} ); } sub print_failed_slaves_if { my $self = shift; my $log = $self->{logger}; my @failed_slaves = $self->get_failed_slaves(); if ( $#failed_slaves >= 0 ) { $log->info("Failed Slaves:"); $self->print_servers( $self->{failed_slaves} ); } } sub print_unmanaged_slaves_if { my $self = shift; my $log = $self->{logger}; my @unmanaged_slaves = $self->get_unmanaged_slaves(); if ( $#unmanaged_slaves >= 0 ) { $log->info("Unmanaged Servers:"); $self->print_servers( $self->{unmanaged_slaves} ); } } sub print_servers { my ( $self, $servers_ref ) = @_; my @servers = @$servers_ref; foreach (@servers) { $_->print_server(); } } sub disconnect_all($) { my $self = shift; my $log = $self->{logger}; my @servers = $self->get_alive_servers(); foreach (@servers) { $_->disconnect(); } } # Check master is not reachable from all alive slaves # prerequisite: all slaves see the same master # return 0;ok 1: running sub is_master_reachable_from_slaves($$) { my $self = shift; my $slaves_ref = shift; my $log = $self->{logger}; my @slaves = $self->get_alive_slaves(); $log->info("Checking the current master is not reachable from all slaves.."); foreach (@slaves) { my $dbhelper = $_->{dbhelper}; $dbhelper->stop_io_thread(); $dbhelper->start_io_thread(); sleep(3); my %status = $dbhelper->check_slave_status(); if ( $status{Status} ne '0' || !defined( $status{Slave_IO_Running} ) ) { $log->error( sprintf( "Got error when stopping/starting io thread on %s", $_->get_hostinfo() ) ); return 1; } if ( $status{Slave_IO_Running} eq "Yes" ) { $log->warning( sprintf( "Master is reachable from slave %s", $_->get_hostinfo() ) ); return 1; } $dbhelper->stop_io_thread(); $log->info( sprintf( " Master is not reachable from slave %s", $_->get_hostinfo() ) ); } $log->info(" done."); return 0; } # checking slave status again before starting main operations. # alive slaves info was already fetched by connect_all_and_read_server_status, # so check_slave_status should not fail here. If it fails, we die here. sub read_slave_status($) { my $self = shift; my $log = $self->{logger}; my @slaves = $self->get_alive_slaves(); $log->debug("Fetching current slave status.."); foreach (@slaves) { my $dbhelper = $_->{dbhelper}; my ($sstatus) = (); my %status = $dbhelper->check_slave_status(); # This should not happen so die if it happens if ( $status{Status} ) { my $msg = "Checking slave status failed."; $msg .= " err=$status{Errstr}" if ( $status{Errstr} ); $log->error($msg); croak; } $_->{latest} = 0; $_->{Master_Log_File} = $status{Master_Log_File}; $_->{Read_Master_Log_Pos} = $status{Read_Master_Log_Pos}; $_->{Relay_Master_Log_File} = $status{Relay_Master_Log_File}; $_->{Exec_Master_Log_Pos} = $status{Exec_Master_Log_Pos}; $_->{Relay_Log_File} = $status{Relay_Log_File}; $_->{Relay_Log_Pos} = $status{Relay_Log_Pos}; $_->{Retrieved_Gtid_Set} = $status{Retrieved_Gtid_Set}; $_->{Executed_Gtid_Set} = $status{Executed_Gtid_Set}; } $log->debug(" Fetching current slave status done."); } sub start_sql_threads_if($) { my $self = shift; my @slaves = $self->get_alive_slaves(); foreach my $slave (@slaves) { $slave->start_sql_thread_if(); } } sub get_failover_advisory_locks($) { my $self = shift; my $log = $self->{logger}; my @slaves = $self->get_alive_slaves(); foreach my $slave (@slaves) { if ( $slave->get_failover_advisory_lock() ) { $log->error( sprintf( "Getting advisory lock failed on %s. Maybe failover script or purge_relay_logs script is running on the same slave?", $slave->get_hostinfo() ) ); croak; } } } sub identify_latest_slaves($$) { my $self = shift; my $find_oldest = shift; $find_oldest = 0 unless ($find_oldest); my $log = $self->{logger}; my @slaves = $self->get_alive_slaves(); my @latest = (); foreach (@slaves) { my $a = $latest[0]{Master_Log_File}; my $b = $latest[0]{Read_Master_Log_Pos}; if ( !$find_oldest && ( ( !$a && !defined($b) ) || ( $_->{Master_Log_File} gt $latest[0]{Master_Log_File} ) || ( ( $_->{Master_Log_File} ge $latest[0]{Master_Log_File} ) && $_->{Read_Master_Log_Pos} > $latest[0]{Read_Master_Log_Pos} ) ) ) { @latest = (); push( @latest, $_ ); } elsif ( $find_oldest && ( ( !$a && !defined($b) ) || ( $_->{Master_Log_File} lt $latest[0]{Master_Log_File} ) || ( ( $_->{Master_Log_File} le $latest[0]{Master_Log_File} ) && $_->{Read_Master_Log_Pos} < $latest[0]{Read_Master_Log_Pos} ) ) ) { @latest = (); push( @latest, $_ ); } elsif ( ( $_->{Master_Log_File} eq $latest[0]{Master_Log_File} ) && ( $_->{Read_Master_Log_Pos} == $latest[0]{Read_Master_Log_Pos} ) ) { push( @latest, $_ ); } } foreach (@latest) { $_->{latest} = 1 if ( !$find_oldest ); $_->{oldest} = 1 if ($find_oldest); } $log->info( sprintf( "The %s binary log file/position on all slaves is" . " %s:%d\n", $find_oldest ? "oldest" : "latest", $latest[0]{Master_Log_File}, $latest[0]{Read_Master_Log_Pos} ) ); if ( $latest[0]{Retrieved_Gtid_Set} ) { $log->info( sprintf( "Retrieved Gtid Set: %s", $latest[0]{Retrieved_Gtid_Set} ) ); } if ($find_oldest) { $self->set_oldest_slaves( \@latest ); } else { $self->set_latest_slaves( \@latest ); } } sub identify_oldest_slaves($) { my $self = shift; return $self->identify_latest_slaves(1); } # 1: higher # -1: older # 0: equal sub pos_cmp { my ( $self, $a_mlf, $a_mlp, $b_mlf, $b_mlp ) = @_; return 0 if ( $a_mlf eq $b_mlf && $a_mlp == $b_mlp ); return -1 if ( $a_mlf lt $b_mlf || ( $a_mlf le $b_mlf && $a_mlp < $b_mlp ) ); return 1; } sub set_no_master_if_older($$$) { my $self = shift; my $mlf = shift; my $mlp = shift; my @slaves = $self->get_alive_slaves(); foreach (@slaves) { $_->{no_master} = 1 if ( $self->pos_cmp( $_->{Master_Log_File}, $_->{Read_Master_Log_Pos}, $mlf, $mlp ) < 0 ); } } sub get_oldest_limit_pos($) { my $self = shift; my @slaves = $self->get_alive_slaves(); my $target; foreach (@slaves) { next if ( $_->{ignore_fail} ); my $a = $target->{Master_Log_File}; my $b = $target->{Read_Master_Log_Pos}; if ( ( !$a && !defined($b) ) || ( $_->{Master_Log_File} lt $target->{Master_Log_File} ) || ( ( $_->{Master_Log_File} le $target->{Master_Log_File} ) && $_->{Read_Master_Log_Pos} < $target->{Read_Master_Log_Pos} ) ) { $target = $_; } } return ( $target->{Master_Log_File}, $target->{Read_Master_Log_Pos} ) if ($target); } sub get_most_advanced_latest_slave($) { my $self = shift; my @latest = $self->get_latest_slaves(); my $target; foreach my $slave (@latest) { $target = $slave unless ($target); if ( $slave->{Relay_Master_Log_File} gt $target->{Relay_Master_Log_File} || ( $slave->{Relay_Master_Log_File} eq $target->{Relay_Master_Log_File} && $slave->{Exec_Master_Log_Pos} > $target->{Exec_Master_Log_Pos} ) ) { $target = $slave; } } return $target; } # check slave is too behind master or not # 0: no or acceptable delay # 1: unacceptable delay (can not be a master) sub check_slave_delay($$$) { my $self = shift; my $target = shift; my $latest = shift; my $log = $self->{logger}; $log->debug( sprintf( "Checking replication delay on %s.. ", $target->get_hostinfo() ) ); if ( ( $latest->{Master_Log_File} gt $target->{Relay_Master_Log_File} ) || ( $latest->{Read_Master_Log_Pos} > $target->{Exec_Master_Log_Pos} + 100000000 ) ) { $log->warning( sprintf( " Slave %s SQL Thread delays too much. Latest log file:%s:%d, Current log file:%s:%d. This server is not selected as a new master because recovery will take long time.\n", $target->get_hostinfo(), $latest->{Master_Log_File}, $latest->{Read_Master_Log_Pos}, $target->{Relay_Master_Log_File}, $target->{Exec_Master_Log_Pos} ) ); return 1; } $log->debug(" ok."); return 0; } # The following servers can not be master: # - dead servers # - Set no_master in conf files (i.e. DR servers) # - log_bin is disabled # - Major version is not the oldest # - too much replication delay sub get_bad_candidate_masters($$$) { my $self = shift; my $latest_slave = shift; my $check_replication_delay = shift; my $log = $self->{logger}; my @servers = $self->get_alive_slaves(); my @ret_servers = (); foreach (@servers) { if ( $_->{no_master} >= 1 || $_->{log_bin} eq '0' || $_->{oldest_major_version} eq '0' || ( $latest_slave && ( $check_replication_delay && $self->check_slave_delay( $_, $latest_slave ) >= 1 ) ) ) { push( @ret_servers, $_ ); } } return @ret_servers; } sub is_target_bad_for_new_master { my $self = shift; my $target = shift; my @bad = $self->get_bad_candidate_masters(); foreach (@bad) { return 1 if ( $target->{id} eq $_->{id} ); } return 0; } # Picking up new master # If preferred node is specified, one of active preferred nodes will be new master. # If the latest server behinds too much (i.e. stopping sql thread for online backups), we should not use it as a new master, but we should fetch relay log there. Even though preferred master is configured, it does not become a master if it's far behind. sub select_new_master { my $self = shift; my $prio_new_master_host = shift; my $prio_new_master_port = shift; my $check_replication_delay = shift; $check_replication_delay = 1 if ( !defined($check_replication_delay) ); my $log = $self->{logger}; my @latest = $self->get_latest_slaves(); my @slaves = $self->get_alive_slaves(); my @pref = $self->get_candidate_masters(); my @bad = $self->get_bad_candidate_masters( $latest[0], $check_replication_delay ); if ( $prio_new_master_host && $prio_new_master_port ) { my $new_master = $self->get_alive_server_by_hostport( $prio_new_master_host, $prio_new_master_port ); if ($new_master) { my $a = $self->get_server_from_by_id( \@bad, $new_master->{id} ); unless ($a) { $log->info("$prio_new_master_host can be new master."); return $new_master; } else { $log->error("$prio_new_master_host is bad as a new master!"); return; } } else { $log->error("$prio_new_master_host is not alive!"); return; } } $log->info("Searching new master from slaves.."); $log->info(" Candidate masters from the configuration file:"); $self->print_servers( \@pref ); $log->info(" Non-candidate masters:"); $self->print_servers( \@bad ); return $latest[0] if ( $#pref < 0 && $#bad < 0 && $latest[0]->{latest_priority} ); if ( $latest[0]->{latest_priority} ) { $log->info( " Searching from candidate_master slaves which have received the latest relay log events.." ) if ( $#pref >= 0 ); foreach my $h (@latest) { foreach my $p (@pref) { if ( $h->{id} eq $p->{id} ) { return $h if ( !$self->get_server_from_by_id( \@bad, $p->{id} ) ); } } } $log->info(" Not found.") if ( $#pref >= 0 ); } #new master is not latest $log->info(" Searching from all candidate_master slaves..") if ( $#pref >= 0 ); foreach my $s (@slaves) { foreach my $p (@pref) { if ( $s->{id} eq $p->{id} ) { my $a = $self->get_server_from_by_id( \@bad, $p->{id} ); return $s unless ($a); } } } $log->info(" Not found.") if ( $#pref >= 0 ); if ( $latest[0]->{latest_priority} ) { $log->info( " Searching from all slaves which have received the latest relay log events.." ); foreach my $h (@latest) { my $a = $self->get_server_from_by_id( \@bad, $h->{id} ); return $h unless ($a); } $log->info(" Not found."); } # none of latest servers can not be a master $log->info(" Searching from all slaves.."); foreach my $s (@slaves) { my $a = $self->get_server_from_by_id( \@bad, $s->{id} ); return $s unless ($a); } $log->info(" Not found."); return; } sub get_new_master_binlog_position($$) { my $self = shift; my $target = shift; # master my $dbhelper = $target->{dbhelper}; my $log = $self->{logger}; $log->info("Getting new master's binlog name and position.."); my ( $file, $pos, $a, $b, $gtid ) = $dbhelper->show_master_status(); if ( $file && defined($pos) ) { $log->info(" $file:$pos"); if ( $self->is_gtid_auto_pos_enabled() ) { $log->info( sprintf( " All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_AUTO_POSITION=1, MASTER_USER='%s', MASTER_PASSWORD='xxx';", ( $target->{hostname} eq $target->{ip} ) ? $target->{hostname} : ("$target->{hostname} or $target->{ip}"), $target->{port}, $target->{repl_user} ) ); } else { $log->info( sprintf( " All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='%s', MASTER_PORT=%d, MASTER_LOG_FILE='%s', MASTER_LOG_POS=%d, MASTER_USER='%s', MASTER_PASSWORD='xxx';", ( $target->{hostname} eq $target->{ip} ) ? $target->{hostname} : ("$target->{hostname} or $target->{ip}"), $target->{port}, $file, $pos, $target->{repl_user} ) ); } } else { $log->error("Getting new master's binlog position failed!"); return; } return ( $file, $pos, $gtid ); } sub change_master_and_start_slave { my ( $self, $target, $master, $master_log_file, $master_log_pos, $log ) = @_; $log = $self->{logger} unless ($log); return if ( $target->{id} eq $master->{id} ); my $dbhelper = $target->{dbhelper}; $log->info( sprintf( " Resetting slave %s and starting replication from the new master %s..", $target->get_hostinfo(), $master->get_hostinfo() ) ); $target->stop_slave($log) unless ( $target->{not_slave} ); $dbhelper->reset_slave() unless ( $target->{not_slave} ); my $addr = $target->{use_ip_for_change_master} ? $master->{ip} : $master->{hostname}; if ( $self->is_gtid_auto_pos_enabled() && !$target->{is_mariadb} ) { $dbhelper->change_master_gtid( $addr, $master->{port}, $master->{repl_user}, $master->{repl_password} ); } else { $dbhelper->change_master( $addr, $master->{port}, $master_log_file, $master_log_pos, $master->{repl_user}, $master->{repl_password} ); } $log->info(" Executed CHANGE MASTER."); # After executing CHANGE MASTER, relay_log_purge is automatically disabled. # If the original value is 0, we should turn to 0 explicitly. if ( !$target->{has_gtid} ) { unless ( $target->{relay_purge} ) { $target->disable_relay_log_purge(); } } my $ret = $target->start_slave($log); unless ($ret) { $log->info(" Slave started."); } return $ret; } sub get_current_alive_master($) { my $self = shift; my $log = $self->{logger}; my $master = $self->get_orig_master(); unless ($master) { $log->error( "MySQL master is not correctly configured. Check master/slave settings"); croak; } my $m = $self->get_alive_server_by_id( $master->{id} ); unless ($m) { $log->warning("MySQL master is not currently alive!"); return; } $log->info( sprintf( "Current Alive Master: %s", $m->get_hostinfo() ) ); return $master; } sub stop_io_threads { my $self = shift; my $log = $self->{logger}; my @alive_slaves = $self->get_alive_slaves(); foreach my $target (@alive_slaves) { $target->stop_io_thread($target); exit 0; } return 0; } sub check_repl_priv { my $self = shift; my @servers = $self->get_alive_servers(); foreach my $target (@servers) { $target->check_repl_priv(); } } sub release_failover_advisory_lock { my $self = shift; my @servers = $self->get_alive_servers(); foreach my $target (@servers) { $target->release_failover_advisory_lock(); } } sub get_current_servers_ascii { my $self = shift; my $orig_master = shift; my @alive_slaves = $self->get_alive_slaves(); my $str = $orig_master->get_hostinfo() . " (current master)"; $str .= " ($orig_master->{node_label})" if ( $orig_master->{node_label} ); $str .= "\n"; foreach my $slave (@alive_slaves) { $str .= " +--" . $slave->get_hostinfo(); $str .= " ($slave->{node_label})" if ( $slave->{node_label} ); $str .= "\n"; } $str .= "\n"; return $str; } sub print_servers_ascii { my $self = shift; my $orig_master = shift; my $log = $self->{logger}; my @alive_slaves = $self->get_alive_slaves(); my $str = "\n"; $str .= $self->get_current_servers_ascii($orig_master); $log->info($str); } sub print_servers_migration_ascii { my $self = shift; my $orig_master = shift; my $new_master = shift; my $orig_master_is_new_slave = shift; my $log = $self->{logger}; my @alive_slaves = $self->get_alive_slaves(); my $str = "\n"; $str .= "From:\n"; $str .= $self->get_current_servers_ascii($orig_master); $str .= "To:\n"; $str .= $new_master->get_hostinfo() . " (new master)"; $str .= " ($new_master->{node_label})" if ( $new_master->{node_label} ); $str .= "\n"; foreach my $slave (@alive_slaves) { next if ( $slave->{id} eq $new_master->{id} ); $str .= " +--" . $slave->get_hostinfo(); $str .= " ($slave->{node_label})" if ( $slave->{node_label} ); $str .= "\n"; } if ($orig_master_is_new_slave) { $str .= " +--" . $orig_master->get_hostinfo(); $str .= " ($orig_master->{node_label})" if ( $orig_master->{node_label} ); $str .= "\n"; } $log->info($str); } # for manual failover/switch only sub manually_decide_new_master { my $self = shift; my $orig_master = shift; my $new_master = shift; my $log = $self->{logger}; printf( "\nStarting master switch from %s to %s? (yes/NO): ", $orig_master->get_hostinfo(), $new_master->get_hostinfo() ); my $ret = ; chomp($ret); if ( lc($ret) !~ /^y/ ) { print "Continue? (yes/NO): "; $ret = ; chomp($ret); if ( lc($ret) !~ /^y/ ) { $orig_master->{not_error} = 1; die "Not typed yes. Stopping."; } print "Enter new master host name: "; $ret = ; chomp($ret); $new_master = $self->get_alive_server_by_hostport( $ret, 3306 ); if ( !$new_master ) { die "New server not found!\n"; } printf "Master switch to %s. OK? (yes/NO): ", $new_master->get_hostinfo(); $ret = ; chomp($ret); die "Not typed yes. Stopping. \n" if ( lc($ret) !~ /^y/ ); } return $new_master; } sub check_replication_health { my $self = shift; my $allow_delay_seconds = shift; $allow_delay_seconds = 1 unless ($allow_delay_seconds); my $log = $self->{logger}; my @alive_slaves = $self->get_alive_slaves(); foreach my $target (@alive_slaves) { $log->info("Checking replication health on $target->{hostname}.."); if ( !$target->current_slave_position() ) { $log->error("Getting slave status failed!"); croak; } if ( $target->has_replication_problem($allow_delay_seconds) ) { $log->error(" failed!"); croak; } else { $log->info(" ok."); } } } sub get_gtid_status($) { my $self = shift; my @servers = $self->get_alive_servers(); my @slaves = $self->get_alive_slaves(); return 0 if ( $#servers < 0 ); foreach (@servers) { return 0 unless ( $_->{has_gtid} ); } foreach (@slaves) { return 0 unless ( $_->{Executed_Gtid_Set} ); } foreach (@slaves) { return 1 if ( defined( $_->{Auto_Position} ) && $_->{Auto_Position} == 1 ); return 1 if ( $_->{use_gtid_auto_pos} ); } return 2; } sub is_gtid_auto_pos_enabled($) { my $self = shift; return 1 if ( $self->{gtid_failover_mode} == 1 ); return 0; } sub force_disable_log_bin_if_auto_pos_disabled($) { my $self = shift; my $log = $self->{logger}; if ( $self->{gtid_failover_mode} == 2 ) { my @slaves = $self->get_alive_slaves(); $log->info("Forcing disable_log_bin since GTID auto pos is disabled"); foreach my $slave (@slaves) { $slave->{disable_log_bin} = 1; } } } sub wait_until_in_sync($$$) { my $self = shift; my $waiter = shift; my $advanced = shift; my $log = $self->{logger}; my $ret; my ( $file, $pos ) = $advanced->get_binlog_position(); $ret = $waiter->master_pos_wait( $file, $pos ); if ($ret) { $log->error("Get error on waiting slave"); } return $ret; } 1; mha4mysql-manager-0.58/rpm/000077500000000000000000000000001325502577300156045ustar00rootroot00000000000000mha4mysql-manager-0.58/rpm/masterha_manager.spec000066400000000000000000000050551325502577300217630ustar00rootroot00000000000000Summary: Master High Availability Manager and Tools for MySQL, Manager Package Name: mha4mysql-manager Version: 0.58 Release: 0%{?dist} License: GPL v2 Vendor: DeNA Co.,Ltd. Group: Manager URL: http://code.google.com/p/mysql-master-ha/ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root BuildArch: noarch BuildRequires: perl(ExtUtils::MakeMaker) >= 6.30 Requires: perl(Config::Tiny) Requires: perl(Log::Dispatch) Requires: perl(Parallel::ForkManager) Requires: mha4mysql-node >= 0.54 Requires: perl(DBD::mysql) Source0: mha4mysql-manager-%{version}.tar.gz %description %{summary}. %prep %setup -q -n mha4mysql-manager-%{version} %build CFLAGS="$RPM_OPT_FLAGS" %{__perl} Makefile.PL INSTALLDIRS="vendor" INSTALLVENDORLIB=%{?perl_install_vendor_lib} make %{?_smp_mflags} OPTIMIZE="$RPM_OPT_FLAGS" %install rm -rf $RPM_BUILD_ROOT make pure_install PERL_INSTALL_ROOT=$RPM_BUILD_ROOT find $RPM_BUILD_ROOT -type f -a \( -name perllocal.pod -o -name .packlist \ -o \( -name '*.bs' -a -empty \) \) -exec rm -f {} ';' find $RPM_BUILD_ROOT -type d -depth -exec rmdir {} 2>/dev/null ';' chmod -R u+w $RPM_BUILD_ROOT/* for brp in %{_prefix}/lib/rpm/%{_build_vendor}/brp-compress \ %{_prefix}/lib/rpm/brp-compress do [ -x $brp ] && $brp && break done find $RPM_BUILD_ROOT -type f \ | sed "s@^$RPM_BUILD_ROOT@@g" \ > %{name}-%{version}-%{release}-filelist if [ "$(cat %{name}-%{version}-%{release}-filelist)X" = "X" ] ; then echo "ERROR: EMPTY FILE LIST" exit 1 fi %clean rm -rf $RPM_BUILD_ROOT %files -f %{name}-%{version}-%{release}-filelist %defattr(-,root,root,-) %changelog * Thu Mar 22 2018 Kenny.Gryp - Several bugfixes - IPV6 support - super_read_only support - enforce_storage_engine support - Version 0.58 - With the help of Marcelo Altmann * Sun May 31 2015 Yoshinori Matsunobu - (Note: All changelogs are written here: http://code.google.com/p/mysql-master-ha/wiki/ReleaseNotes ) - Version 0.57 * Tue Apr 1 2014 Yoshinori Matsunobu - Version 0.56 * Wed Dec 12 2012 Yoshinori Matsunobu - Version 0.55 * Sat Dec 1 2012 Yoshinori Matsunobu - Version 0.54 * Mon Jan 9 2012 Yoshinori Matsunobu - Version 0.53 * Fri Sep 16 2011 Yoshinori Matsunobu - Version 0.52 * Thu Aug 18 2011 Yoshinori Matsunobu - Version 0.51 * Sat Jul 23 2011 Yoshinori Matsunobu - Version 0.50 mha4mysql-manager-0.58/samples/000077500000000000000000000000001325502577300164525ustar00rootroot00000000000000mha4mysql-manager-0.58/samples/conf/000077500000000000000000000000001325502577300173775ustar00rootroot00000000000000mha4mysql-manager-0.58/samples/conf/app1.cnf000066400000000000000000000004011325502577300207230ustar00rootroot00000000000000[server default] manager_workdir=/var/log/masterha/app1 manager_log=/var/log/masterha/app1/manager.log [server1] hostname=host1 candidate_master=1 [server2] hostname=host2 candidate_master=1 [server3] hostname=host3 [server4] hostname=host4 no_master=1 mha4mysql-manager-0.58/samples/conf/masterha_default.cnf000066400000000000000000000007331325502577300234020ustar00rootroot00000000000000[server default] user=root password=rootpass ssh_user=root master_binlog_dir= /var/lib/mysql,/var/log/mysql remote_workdir=/data/log/masterha secondary_check_script= masterha_secondary_check -s remote_host1 -s remote_host2 ping_interval=3 # master_ip_failover_script= /script/masterha/master_ip_failover # shutdown_script= /script/masterha/power_manager # report_script= /script/masterha/send_report # master_ip_online_change_script= /script/masterha/master_ip_online_change mha4mysql-manager-0.58/samples/scripts/000077500000000000000000000000001325502577300201415ustar00rootroot00000000000000mha4mysql-manager-0.58/samples/scripts/master_ip_failover000077500000000000000000000071001325502577300237370ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ## Note: This is a sample script and is not complete. Modify the script based on your environment. use strict; use warnings FATAL => 'all'; use Getopt::Long; use MHA::DBHelper; my ( $command, $ssh_user, $orig_master_host, $orig_master_ip, $orig_master_port, $new_master_host, $new_master_ip, $new_master_port, $new_master_user, $new_master_password ); GetOptions( 'command=s' => \$command, 'ssh_user=s' => \$ssh_user, 'orig_master_host=s' => \$orig_master_host, 'orig_master_ip=s' => \$orig_master_ip, 'orig_master_port=i' => \$orig_master_port, 'new_master_host=s' => \$new_master_host, 'new_master_ip=s' => \$new_master_ip, 'new_master_port=i' => \$new_master_port, 'new_master_user=s' => \$new_master_user, 'new_master_password=s' => \$new_master_password, ); exit &main(); sub main { if ( $command eq "stop" || $command eq "stopssh" ) { # $orig_master_host, $orig_master_ip, $orig_master_port are passed. # If you manage master ip address at global catalog database, # invalidate orig_master_ip here. my $exit_code = 1; eval { # updating global catalog, etc $exit_code = 0; }; if ($@) { warn "Got Error: $@\n"; exit $exit_code; } exit $exit_code; } elsif ( $command eq "start" ) { # all arguments are passed. # If you manage master ip address at global catalog database, # activate new_master_ip here. # You can also grant write access (create user, set read_only=0, etc) here. my $exit_code = 10; eval { my $new_master_handler = new MHA::DBHelper(); # args: hostname, port, user, password, raise_error_or_not $new_master_handler->connect( $new_master_ip, $new_master_port, $new_master_user, $new_master_password, 1 ); ## Set read_only=0 on the new master $new_master_handler->disable_log_bin_local(); print "Set read_only=0 on the new master.\n"; $new_master_handler->disable_read_only(); ## Creating an app user on the new master print "Creating app user on the new master..\n"; FIXME_xxx_create_user( $new_master_handler->{dbh} ); $new_master_handler->enable_log_bin_local(); $new_master_handler->disconnect(); ## Update master ip on the catalog database, etc FIXME_xxx; $exit_code = 0; }; if ($@) { warn $@; # If you want to continue failover, exit 10. exit $exit_code; } exit $exit_code; } elsif ( $command eq "status" ) { # do nothing exit 0; } else { &usage(); exit 1; } } sub usage { print "Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n"; } mha4mysql-manager-0.58/samples/scripts/master_ip_online_change000077500000000000000000000232161325502577300247270ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ## Note: This is a sample script and is not complete. Modify the script based on your environment. use strict; use warnings FATAL => 'all'; use Getopt::Long; use MHA::DBHelper; use MHA::NodeUtil; use Time::HiRes qw( sleep gettimeofday tv_interval ); use Data::Dumper; my $_tstart; my $_running_interval = 0.1; my ( $command, $orig_master_is_new_slave, $orig_master_host, $orig_master_ip, $orig_master_port, $orig_master_user, $orig_master_password, $orig_master_ssh_user, $new_master_host, $new_master_ip, $new_master_port, $new_master_user, $new_master_password, $new_master_ssh_user, ); GetOptions( 'command=s' => \$command, 'orig_master_is_new_slave' => \$orig_master_is_new_slave, 'orig_master_host=s' => \$orig_master_host, 'orig_master_ip=s' => \$orig_master_ip, 'orig_master_port=i' => \$orig_master_port, 'orig_master_user=s' => \$orig_master_user, 'orig_master_password=s' => \$orig_master_password, 'orig_master_ssh_user=s' => \$orig_master_ssh_user, 'new_master_host=s' => \$new_master_host, 'new_master_ip=s' => \$new_master_ip, 'new_master_port=i' => \$new_master_port, 'new_master_user=s' => \$new_master_user, 'new_master_password=s' => \$new_master_password, 'new_master_ssh_user=s' => \$new_master_ssh_user, ); exit &main(); sub current_time_us { my ( $sec, $microsec ) = gettimeofday(); my $curdate = localtime($sec); return $curdate . " " . sprintf( "%06d", $microsec ); } sub sleep_until { my $elapsed = tv_interval($_tstart); if ( $_running_interval > $elapsed ) { sleep( $_running_interval - $elapsed ); } } sub get_threads_util { my $dbh = shift; my $my_connection_id = shift; my $running_time_threshold = shift; my $type = shift; $running_time_threshold = 0 unless ($running_time_threshold); $type = 0 unless ($type); my @threads; my $sth = $dbh->prepare("SHOW PROCESSLIST"); $sth->execute(); while ( my $ref = $sth->fetchrow_hashref() ) { my $id = $ref->{Id}; my $user = $ref->{User}; my $host = $ref->{Host}; my $command = $ref->{Command}; my $state = $ref->{State}; my $query_time = $ref->{Time}; my $info = $ref->{Info}; $info =~ s/^\s*(.*?)\s*$/$1/ if defined($info); next if ( $my_connection_id == $id ); next if ( defined($query_time) && $query_time < $running_time_threshold ); next if ( defined($command) && $command eq "Binlog Dump" ); next if ( defined($user) && $user eq "system user" ); next if ( defined($command) && $command eq "Sleep" && defined($query_time) && $query_time >= 1 ); if ( $type >= 1 ) { next if ( defined($command) && $command eq "Sleep" ); next if ( defined($command) && $command eq "Connect" ); } if ( $type >= 2 ) { next if ( defined($info) && $info =~ m/^select/i ); next if ( defined($info) && $info =~ m/^show/i ); } push @threads, $ref; } return @threads; } sub main { if ( $command eq "stop" ) { ## Gracefully killing connections on the current master # 1. Set read_only= 1 on the new master # 2. DROP USER so that no app user can establish new connections # 3. Set read_only= 1 on the current master # 4. Kill current queries # * Any database access failure will result in script die. my $exit_code = 1; eval { ## Setting read_only=1 on the new master (to avoid accident) my $new_master_handler = new MHA::DBHelper(); # args: hostname, port, user, password, raise_error(die_on_error)_or_not $new_master_handler->connect( $new_master_ip, $new_master_port, $new_master_user, $new_master_password, 1 ); print current_time_us() . " Set read_only on the new master.. "; $new_master_handler->enable_read_only(); if ( $new_master_handler->is_read_only() ) { print "ok.\n"; } else { die "Failed!\n"; } $new_master_handler->disconnect(); # Connecting to the orig master, die if any database error happens my $orig_master_handler = new MHA::DBHelper(); $orig_master_handler->connect( $orig_master_ip, $orig_master_port, $orig_master_user, $orig_master_password, 1 ); ## Drop application user so that nobody can connect. Disabling per-session binlog beforehand $orig_master_handler->disable_log_bin_local(); print current_time_us() . " Drpping app user on the orig master..\n"; FIXME_xxx_drop_app_user($orig_master_handler); ## Waiting for N * 100 milliseconds so that current connections can exit my $time_until_read_only = 15; $_tstart = [gettimeofday]; my @threads = get_threads_util( $orig_master_handler->{dbh}, $orig_master_handler->{connection_id} ); while ( $time_until_read_only > 0 && $#threads >= 0 ) { if ( $time_until_read_only % 5 == 0 ) { printf "%s Waiting all running %d threads are disconnected.. (max %d milliseconds)\n", current_time_us(), $#threads + 1, $time_until_read_only * 100; if ( $#threads < 5 ) { print Data::Dumper->new( [$_] )->Indent(0)->Terse(1)->Dump . "\n" foreach (@threads); } } sleep_until(); $_tstart = [gettimeofday]; $time_until_read_only--; @threads = get_threads_util( $orig_master_handler->{dbh}, $orig_master_handler->{connection_id} ); } ## Setting read_only=1 on the current master so that nobody(except SUPER) can write print current_time_us() . " Set read_only=1 on the orig master.. "; $orig_master_handler->enable_read_only(); if ( $orig_master_handler->is_read_only() ) { print "ok.\n"; } else { die "Failed!\n"; } ## Waiting for M * 100 milliseconds so that current update queries can complete my $time_until_kill_threads = 5; @threads = get_threads_util( $orig_master_handler->{dbh}, $orig_master_handler->{connection_id} ); while ( $time_until_kill_threads > 0 && $#threads >= 0 ) { if ( $time_until_kill_threads % 5 == 0 ) { printf "%s Waiting all running %d queries are disconnected.. (max %d milliseconds)\n", current_time_us(), $#threads + 1, $time_until_kill_threads * 100; if ( $#threads < 5 ) { print Data::Dumper->new( [$_] )->Indent(0)->Terse(1)->Dump . "\n" foreach (@threads); } } sleep_until(); $_tstart = [gettimeofday]; $time_until_kill_threads--; @threads = get_threads_util( $orig_master_handler->{dbh}, $orig_master_handler->{connection_id} ); } ## Terminating all threads print current_time_us() . " Killing all application threads..\n"; $orig_master_handler->kill_threads(@threads) if ( $#threads >= 0 ); print current_time_us() . " done.\n"; $orig_master_handler->enable_log_bin_local(); $orig_master_handler->disconnect(); ## After finishing the script, MHA executes FLUSH TABLES WITH READ LOCK $exit_code = 0; }; if ($@) { warn "Got Error: $@\n"; exit $exit_code; } exit $exit_code; } elsif ( $command eq "start" ) { ## Activating master ip on the new master # 1. Create app user with write privileges # 2. Moving backup script if needed # 3. Register new master's ip to the catalog database # We don't return error even though activating updatable accounts/ip failed so that we don't interrupt slaves' recovery. # If exit code is 0 or 10, MHA does not abort my $exit_code = 10; eval { my $new_master_handler = new MHA::DBHelper(); # args: hostname, port, user, password, raise_error_or_not $new_master_handler->connect( $new_master_ip, $new_master_port, $new_master_user, $new_master_password, 1 ); ## Set read_only=0 on the new master $new_master_handler->disable_log_bin_local(); print current_time_us() . " Set read_only=0 on the new master.\n"; $new_master_handler->disable_read_only(); ## Creating an app user on the new master print current_time_us() . " Creating app user on the new master..\n"; FIXME_xxx_create_app_user($new_master_handler); $new_master_handler->enable_log_bin_local(); $new_master_handler->disconnect(); ## Update master ip on the catalog database, etc $exit_code = 0; }; if ($@) { warn "Got Error: $@\n"; exit $exit_code; } exit $exit_code; } elsif ( $command eq "status" ) { # do nothing exit 0; } else { &usage(); exit 1; } } sub usage { print "Usage: master_ip_online_change --command=start|stop|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n"; die; } mha4mysql-manager-0.58/samples/scripts/power_manager000077500000000000000000000271331325502577300227230ustar00rootroot00000000000000#!/usr/bin/env perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ## Note: This is a sample script and is not complete. Modify the script based on your environment. use strict; use warnings FATAL => 'all'; use Getopt::Long; use Pod::Usage; use Net::Telnet; use MHA::ManagerConst; use MHA::ManagerUtil; my $SSH_STOP_OK = 10; my $COMMAND_NOT_SUPPORTED = 20; my $ILO_ADMIN = 'Administrator'; my $DRAC_ADMIN = 'root'; my $PASSWORD = 'xxx'; my $max_retries = 10; exit &main(); sub get_power_status_drac_internal { my $telnet = shift; my $prompt = shift; $telnet->print("racadm serveraction powerstatus"); ($_) = $telnet->waitfor($prompt); my $power_state = "void"; my @cmd_out = split /\n/; # discard command sent to DRAC $_ = shift @cmd_out; #strip ansi control chars s/\e\[(([0-9]+;)*[0-9]+)*[ABCDfHJKmsu]//g; s/^.*\x0D//; foreach (@cmd_out) { s/^\s+//g; s/\s+$//g; if (m/^Server power status: (\w+)/) { $power_state = lc($1); last; } } return $power_state; } sub power_off_drac_internal { my $telnet = shift; my $prompt = shift; $telnet->print("racadm serveraction powerdown"); $telnet->waitfor($prompt); } sub power_on_drac_internal { my $telnet = shift; my $prompt = shift; $telnet->print("racadm serveraction powerup"); $telnet->waitfor($prompt); } sub login_drac_internal { my $drac_addr = shift; my $prompt = '/admin1|\$/'; my $telnet = new Net::Telnet( Timeout => 10, Prompt => $prompt, ); $telnet->open($drac_addr); $telnet->waitfor('/login/i'); $telnet->print($DRAC_ADMIN); $telnet->waitfor('/password/i'); $telnet->print($PASSWORD); $telnet->waitfor($prompt); return ( $telnet, $prompt ); } sub power_off_drac { my $drac_addr = shift; my $power_status = "void"; local $@; eval { my ( $telnet, $prompt ) = login_drac_internal($drac_addr); power_off_drac_internal( $telnet, $prompt ); $power_status = get_power_status_drac_internal( $telnet, $prompt ); $telnet->close; }; if ($@) { warn $@; } return $power_status; } sub power_on_drac { my $drac_addr = shift; my $power_status = "void"; local $@; eval { my ( $telnet, $prompt ) = login_drac_internal($drac_addr); power_on_drac_internal( $telnet, $prompt ); $power_status = get_power_status_drac_internal( $telnet, $prompt ); $telnet->close; }; if ($@) { warn $@; } return $power_status; } sub power_status_drac { my $drac_addr = shift; my $power_status = "void"; local $@; eval { my ( $telnet, $prompt ) = login_drac_internal($drac_addr); $power_status = get_power_status_drac_internal( $telnet, $prompt ); $telnet->close; }; if ($@) { warn $@; } return $power_status; } sub power_status_ilo { my $ilo_addr = shift; my $power_status = "void"; local $@; eval { my $ipmi_out = `ipmitool -H $ilo_addr -U $ILO_ADMIN -P $PASSWORD -I lanplus power status`; die "Failed to get power status from ipmitool. Maybe you need to upgrade ILO firmware version.\n" if ($?); chomp($ipmi_out); if ( $ipmi_out =~ m/^Chassis Power is (\w+)/ ) { $power_status = lc($1); } }; if ($@) { warn $@; } return $power_status; } sub power_on_ilo { my $ilo_addr = shift; my $power_status = "void"; local $@; eval { $power_status = power_status_ilo($ilo_addr); if ( $power_status ne "off" ) { die "Power from ipmitool is already on.\n" if ( $power_status eq "on" ); return $power_status; } `ipmitool -H $ilo_addr -U $ILO_ADMIN -P $PASSWORD -I lanplus power on`; $power_status = power_status_ilo($ilo_addr); }; if ($@) { warn $@; } return $power_status; } sub power_off_ilo { my $ilo_addr = shift; my $power_status = "void"; local $@; eval { $power_status = power_status_ilo($ilo_addr); if ( $power_status ne "on" ) { die "Power from ipmitool is already off.\n" if ( $power_status eq "off" ); return $power_status; } `ipmitool -H $ilo_addr -U $ILO_ADMIN -P $PASSWORD -I lanplus power off`; $power_status = power_status_ilo($ilo_addr); }; if ($@) { warn $@; } return $power_status; } sub get_power_status { my ( $admin_addr, $server_type ) = @_; my $power_status = "void"; if ( $server_type eq "ilo" ) { $power_status = power_status_ilo($admin_addr); } elsif ( $server_type eq "drac" ) { $power_status = power_status_drac($admin_addr); } return $power_status; } sub stop { my ( $real_host, $admin_addr, $server_type ) = @_; my $power_status = "void"; if ( $server_type eq "ilo" ) { $power_status = power_off_ilo($admin_addr); } elsif ( $server_type eq "drac" ) { $power_status = power_off_drac($admin_addr); } if ( $power_status eq "off" ) { print "Power of $real_host was successfully turned off.\n"; return 0; } elsif ( $power_status ne "on" ) { return $COMMAND_NOT_SUPPORTED; } my $retry_count = 0; while ( $retry_count < $max_retries ) { $power_status = get_power_status( $admin_addr, $server_type ); last if ( $power_status eq "off" ); print "Waiting until power status becomes 'off'. Current status is $power_status ...\n"; sleep 3; $retry_count++; } if ( $power_status eq "off" ) { print "Power of $real_host was successfully turned off.\n"; return 0; } else { print "Power of $real_host was not turned off. Check the host for detail.\n"; return 1; } } sub stopssh { my ( $ssh_user, $real_host, $real_ip, $pid_file ) = @_; my $ssh_user_host = $ssh_user . '@'; if ($real_ip) { $ssh_user_host .= $real_ip; } else { $ssh_user_host .= $real_host; } my $command; my ( $high_ret, $low_ret ); if ($pid_file) { $command = "\"if [ ! -e $pid_file ]; then exit 1; fi; pid=\\\`cat $pid_file\\\`; rm -f $pid_file; kill -9 \\\$pid; a=\\\`ps ax | grep $pid_file | grep -v grep | wc | awk {'print \\\$1'}\\\`; if [ \"a\\\$a\" = \"a0\" ]; then exit 10; fi; sleep 1; a=\\\`ps ax | grep $pid_file | grep -v grep | wc | awk {'print \\\$1'}\\\`; if [ \"a\\\$a\" = \"a0\" ]; then exit 10; else exit 1; fi\""; ( $high_ret, $low_ret ) = MHA::ManagerUtil::exec_system( "ssh $ssh_user_host $MHA::ManagerConst::SSH_OPT_CHECK $command"); if ( $high_ret == $SSH_STOP_OK && $low_ret == 0 ) { print "ssh reachable. mysqld stopped. power off not needed.\n"; return $high_ret; } print "Killing mysqld instance based on $pid_file failed.\n"; } print "Killing all mysqld instances on $real_host..\n"; $command = "\"killall -9 mysqld mysqld_safe; a=\\\`pidof mysqld\\\`; if [ \\\"a\\\$a\\\" = \\\"a\\\" ]; then exit 10; fi; sleep 1; a=\\\`pidof mysqld\\\`; if [ \\\"a\\\$a\\\" = \\\"a\\\" ]; then exit 10; else exit 1; fi\""; ( $high_ret, $low_ret ) = MHA::ManagerUtil::exec_system( "ssh $ssh_user_host $MHA::ManagerConst::SSH_OPT_CHECK $command"); if ( $high_ret == $SSH_STOP_OK && $low_ret == 0 ) { print "ssh reachable. mysqld stopped. power off not needed.\n"; return $high_ret; } else { print "ssh NOT reachable. Power off needed (rc1=$high_ret, rc2=$low_ret).\n"; return 1; } } sub start { my ( $real_host, $admin_addr, $server_type ) = @_; my $power_status = "void"; if ( $server_type eq "ilo" ) { $power_status = power_on_ilo($admin_addr); } elsif ( $server_type eq "drac" ) { $power_status = power_on_drac($admin_addr); } if ( $power_status eq "on" ) { print "Power of $real_host was successfully turned on.\n"; return 0; } elsif ( $power_status ne "off" ) { return $COMMAND_NOT_SUPPORTED; } my $retry_count = 0; while ( $power_status ne "on" && $retry_count < $max_retries ) { $power_status = get_power_status( $admin_addr, $server_type ); last if ( $power_status eq "on" ); print "Waiting until power status becomes 'on'. Current status is $power_status ...\n"; sleep 3; $retry_count++; } if ( $power_status eq "on" ) { print "Power of $real_host was successfully turned on.\n"; return 0; } else { print "Power of $real_host was not turned on. Check the host for detail.\n"; return 1; } } sub status { my ( $real_host, $admin_addr, $server_type ) = @_; my $power_status = get_power_status( $admin_addr, $server_type ); print "Current power status on $real_host : $power_status\n"; if ( $power_status eq "on" ) { return 0; } elsif ( $power_status eq "off" ) { return 0; } else { return $COMMAND_NOT_SUPPORTED; } } # If ssh is reachable and mysqld process does not exist, exit with 2 and # do not power off. If ssh is not reachable, do power off and exit with 0 # if successful. Otherwise exit with 1. sub main { my ( $command, $ssh_user, $host, $ip, $port, $pid_file, $help ); GetOptions( 'command=s' => \$command, 'ssh_user=s' => \$ssh_user, 'host=s' => \$host, 'ip=s' => \$ip, 'port=i' => \$port, 'pid_file=s' => \$pid_file, 'help' => \$help, ); if ($help) { pod2usage(0); } pod2usage(1) unless ($command); my $rc = 1; my $ssh_stop_fail = 0; if ( $command eq "stopssh" || $command eq "stopssh2" ) { pod2usage(1) unless ($ssh_user); pod2usage(1) unless ($host); $rc = stopssh( $ssh_user, $host, $ip, $pid_file ); if ( $rc == $SSH_STOP_OK ) { exit $rc; } else { exit 1 if ( $command eq "stopssh2" ); $ssh_stop_fail = 1; } } # Get server type (ilo/drac, etc) and administrative IP address. my ( $admin_addr, $server_type ) = FIXME_xxx( $host, $ip ); if ( $command eq "start" ) { $rc = start( $host, $admin_addr, $server_type ); } elsif ( $command eq "stop" || $ssh_stop_fail ) { $rc = stop( $host, $admin_addr, $server_type ); } elsif ( $command eq "status" ) { $rc = status( $host, $admin_addr, $server_type ); } else { pod2usage(1); } # Do other way to stop host if ( $rc == $COMMAND_NOT_SUPPORTED ) { $rc = FIXME_xxx( $command, $host, $ip ); } if ( $rc == 0 ) { exit 0; } else { exit 1; } } ############################################################################# =head1 NAME Main purpose of this command is node fencing so that split brain never happens. =head1 SYNOPSIS # power off power_manager --command=stop --host=master_server # killing mysqld and mysqld_safe at first. If not successful, forcing power off power_manager --command=stopssh --host=master_server --ssh_user=root # killing mysqld and mysqld_safe. If not successful, just exit. power_manager --command=stopssh2 --host=master_server --ssh_user=root # killing mysqld with specified pid file. This is useful when you run multiple MySQL instances and want to stop only specified instance power_manager --command=stopssh --host=master_server --ssh_user=root --pid_file=/var/lib/mysql/mysqld.pid # power on power_manager --command=start --host=master_server # checking power status power_manager --command=status --host=master_server mha4mysql-manager-0.58/samples/scripts/send_report000077500000000000000000000025201325502577300224120ustar00rootroot00000000000000#!/usr/bin/perl # Copyright (C) 2011 DeNA Co.,Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ## Note: This is a sample script and is not complete. Modify the script based on your environment. use strict; use warnings FATAL => 'all'; use Getopt::Long; #new_master_host and new_slave_hosts are set only when recovering master succeeded my ( $dead_master_host, $new_master_host, $new_slave_hosts, $subject, $body ); GetOptions( 'orig_master_host=s' => \$dead_master_host, 'new_master_host=s' => \$new_master_host, 'new_slave_hosts=s' => \$new_slave_hosts, 'subject=s' => \$subject, 'body=s' => \$body, ); # Do whatever you want here exit 0; mha4mysql-manager-0.58/t/000077500000000000000000000000001325502577300152515ustar00rootroot00000000000000mha4mysql-manager-0.58/t/99-perlcritic.t000066400000000000000000000003311325502577300200320ustar00rootroot00000000000000use strict; use Test::More; eval { require Test::Perl::Critic; Test::Perl::Critic->import(-profile => "t/perlcriticrc") }; plan skip_all => "Test::Perl::Critic is not installed." if $@; all_critic_ok("bin", "lib"); mha4mysql-manager-0.58/t/perlcriticrc000066400000000000000000000001201325502577300176520ustar00rootroot00000000000000# allow prototypes in soubroutines [-Subroutines::ProhibitSubroutinePrototypes] mha4mysql-manager-0.58/tests/000077500000000000000000000000001325502577300161505ustar00rootroot00000000000000mha4mysql-manager-0.58/tests/intro.txt000066400000000000000000000006041325502577300200440ustar00rootroot00000000000000MHA public test cases maatkit (mk-table-checksum), MySQL Sandbox, and MySQL binary packages(that are used from MySQL Sandbox) are needed to run tests. Edit t/env.sh and modify some variables (i.e. SANDBOX_HOME) as you like. By default, SANDBOX_HOME is /opt/mysql/sandbox_data. In this case, you need to put target MySQL binary tarball under /opt/mysql/X.Y.Z (i.e. /opt/mysql/5.5.15). mha4mysql-manager-0.58/tests/run_suites.sh000077500000000000000000000015261325502577300207130ustar00rootroot00000000000000#!/bin/sh cd t echo "5.5.16 row" ./run_tests --mysql_version=5.5.16 --use_row_format echo "5.5.16 stmt" ./run_tests --mysql_version=5.5.16 echo "5.5.16 stmt relay log info" ./run_tests --mysql_version=5.5.16 --init_script=change_relay_log_info.sh --tests=need* ./run_tests --mysql_version=5.5.16 --init_script=change_relay_log_info.sh --tests=large* echo "5.5.15 stmt" ./run_tests --mysql_version=5.5.15 echo "5.1.58 row" ./run_tests --mysql_version=5.1.58 --use_row_format echo "5.1.58 stmt" ./run_tests --mysql_version=5.1.58 echo "5.1.58 stmt relay log info" ./run_tests --mysql_version=5.1.58 --init_script=change_relay_log_info.sh --tests=need* ./run_tests --mysql_version=5.1.58 --init_script=change_relay_log_info.sh --tests=large* echo "5.0.91" ./run_tests --mysql_version=5.0.91 echo "5.0.45" ./run_tests --mysql_version=5.0.45 mha4mysql-manager-0.58/tests/t/000077500000000000000000000000001325502577300164135ustar00rootroot00000000000000mha4mysql-manager-0.58/tests/t/bulk_tran_insert.pl000077500000000000000000000023311325502577300223170ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings; use DBI; my ( $port, $user, $pass, $start, $stop, $commit_interval ) = @ARGV; my $pk = $start; my $committed_rows = 0; my $dsn = "DBI:mysql:test;host=127.0.0.1;port=$port"; my $dbh = DBI->connect( $dsn, $user, $pass ); if ( $commit_interval == 1 ) { $dbh->do("SET AUTOCOMMIT=1"); } else { $dbh->do("SET AUTOCOMMIT=0"); } while ( $pk <= $stop ) { my ( $sth, $ret ); $sth = $dbh->prepare(<<'SQL'); insert into t1 values(?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?), (?, ?, ?) SQL eval { my @params = (); for ( my $j = 0 ; $j < 10 ; $j++ ) { my $val = $pk + $j; my $val2 = 'aaaaaa' . $pk; push @params, $val; push @params, $val; push @params, $val2; } $ret = $sth->execute(@params); die if ( $ret != 10 ); }; if ($@) { return 1; } $pk = $pk + 10; if ( $commit_interval == 1 ) { $committed_rows = $committed_rows + 10; } elsif ( $commit_interval > 1 && ( $pk - 1 ) % $commit_interval == 0 ) { eval { $dbh->do("COMMIT"); }; if ($@) { return 1; } $committed_rows += $commit_interval; } } mha4mysql-manager-0.58/tests/t/change_relay_log_info.sh000077500000000000000000000004041325502577300232450ustar00rootroot00000000000000#!/bin/sh ./stop_s1.sh ./stop_s2.sh rm -rf /var/tmp/mha_relay_test_dir mkdir -p /var/tmp/mha_relay_test_dir ./start_s1.sh --relay_log_info=/var/tmp/mha_relay_test_dir/relay_log.info ./start_s2.sh --relay_log_info=/var/tmp/mha_relay_test_dir/relay_log2.info mha4mysql-manager-0.58/tests/t/check000077500000000000000000000021041325502577300174130ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings FATAL=>'all'; my $testcase_name=$ARGV[0]; my $expected_records=$ARGV[1]; my $expected_hosts=$ARGV[2]; sleep 2; unless($expected_hosts) { $expected_hosts="h=127.0.0.1,P=10001 h=127.0.0.1,P=10002 h=127.0.0.1,P=10003 h=127.0.0.1,P=10004" } `mk-table-checksum --databases=test $expected_hosts --count --trim > checksum.log`; my $checksum_uniq=`cat checksum.log | awk '{print \$7}' | grep -v CHECKSUM | sort -u | wc | awk '{print \$1}'`; chomp($checksum_uniq); if($checksum_uniq ne '1') { open (FH, ">>checksum.log"); print FH "Checksum is not unique!\n"; close(FH); &handle_fail(); } my $num_rows=`cat checksum.log | awk '{print \$6}' | grep -v COUNT | head -n 1`; chomp($num_rows); if($num_rows ne $expected_records) { open (FH, ">>checksum.log"); print FH "Number of records $num_rows is not equal to expected records $expected_records!\n"; close(FH); &handle_fail(); } &handle_ok(); sub handle_fail{ print "$testcase_name [Fail (checksum)]\n"; exit 1; } sub handle_ok{ print "$testcase_name [Pass]\n"; exit 0; } mha4mysql-manager-0.58/tests/t/env.sh000077500000000000000000000117421325502577300175470ustar00rootroot00000000000000#!/bin/sh if [ "A$VERSION" = "A" ]; then export VERSION=5.5.16 fi export VERSION_DIR=`echo $VERSION | perl -pi -e 's/\./\_/g'` if [ "A$USE_ROW_FORMAT" = "A" ]; then #export MASTER_OPTIONS="--my_file=my.cnf" #export SLAVE_OPTIONS="--my_file=my-gtid-slave.cnf" export NODE_OPTIONS="--my_file=my.cnf" else export NODE_OPTIONS="--my_file=my-row.cnf" fi export SANDBOX_HOME=/opt/mysql/sandbox_data export SANDBOX_AS_ROOT=1 export MYSQL_USER=root export MYSQL_PWD=msandbox export M="--host=127.0.0.1 --port=10000" export S1="--host=127.0.0.1 --port=10001" export S2="--host=127.0.0.1 --port=10002" export S3="--host=127.0.0.1 --port=10003" export S4="--host=127.0.0.1 --port=10004" export MP=10000 export S1P=10001 export S2P=10002 export S3P=10003 export S4P=10004 export CONF=mha_test.cnf export CONF_LATEST=mha_test_latest.cnf export CONF_IGNORE=mha_test_ignore.cnf export CONF_BINLOG=mha_test_binlog.cnf export MASTER_DATA_DIR=$SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data/ export CLIENT_BINDIR="" export CLIENT_LIBDIR="" if [ "A$CUSTOM_CLIENTS" = "Ayes" ]; then export CLIENT_BINDIR="client_bindir=/opt/mysql/$VERSION/bin" # export CLIENT_LIBDIR="client_libdir=/opt/mysql/$VERSION/lib/mysql" elif [ "A$CUSTOM_CLIENTS" = "Abad" ]; then export CLIENT_BINDIR="client_bindir=/opt/mysql/$VERSION" export CLIENT_LIBDIR="client_libdir=/opt/mysql/$VERSION" else export CLIENT_BINDIR="" export CLIENT_LIBDIR="" fi fail_if_zero() { if test $2 -eq 0 ; then echo "$1 [Fail] (expected non-zero exit code, but $2 returned)" exit 1 fi } fail_if_nonzero() { if test $2 -ne 0 ; then echo "$1 [Fail] (expected zero exit code, but $2 returned)" exit 1 fi } fail_if_nonN() { if test $2 -ne $3 ; then echo "$1 [Fail] (expected $3 exit code, but $2 returned)" exit 1 fi } fail_if_empty() { if test ! -s $2 ; then echo "$1 [Fail] ($2 is empty)" exit 1 fi } fail_if_nonempty() { if test -s $2 ; then echo "$1 [Fail] ($2 is not empty)" exit 1 fi } skip_if_gtid() { GTID=`mysql -h 127.0.0.1 --port=$S3P -e "show slave status\G" | grep Auto_Position | awk {'print $2'}` if [ "a$GTID" = "a1" ]; then echo "$1 [Skip]" exit 0 fi } skip_if_not_gtid() { GTID=`mysql -h 127.0.0.1 --port=$S3P -e "show slave status\G" | grep Auto_Position | awk {'print $2'}` if [ "a$GTID" = "a1" ]; then return 0 else echo "$1 [Skip]" exit 0 fi } is_gtid_supported() { GTID=`mysql -h 127.0.0.1 --port=$S3P -e "show slave status\G" | grep Auto_Position | awk {'print $2'}` if [ "a$GTID" = "a1" ]; then return 1 fi return 0 } is_read_only() { READ_ONLY=`mysql -h127.0.0.1 --port=$2 -e "select @@global.read_only\G" | grep read_only | awk '{print $2}'` if [ "$READ_ONLY" = "1" ]; then return else echo "$1 [Fail (read_only is 0)]" exit 1 fi } check_sql_yes() { SQL_STATUS=`mysql -h127.0.0.1 --port=$2 -e "show slave status\G" | grep Slave_SQL_Running: | awk '{print $2}'` if [ "$SQL_STATUS" = "Yes" ]; then return else echo "$1 [Fail (slave not running)]" exit 1 fi } check_sql_stop() { SQL_STATUS=`mysql -h127.0.0.1 --port=$2 -e "show slave status\G" | grep Slave_SQL_Running: | awk '{print $2}'` if [ "$SQL_STATUS" = "No" ]; then return else echo "$1 [Fail (slave not stop)]" exit 1 fi } check_master() { MASTER_PORT=`mysql -h127.0.0.1 --port=$2 -e "show slave status\G" | grep Master_Port: | awk '{print $2}'` if [ "$MASTER_PORT" = "$3" ]; then return else echo "$1 [Fail (Master Port $2 is not equal to $3)]" exit 1 fi } get_binlog_file() { B_FILE=`mysql -h127.0.0.1 --port=$1 -e "show master status\G" | grep File | awk {'print $2'}` echo $B_FILE } get_binlog_position() { POS=`mysql -h127.0.0.1 --port=$1 -e "show master status\G" | grep Position | awk {'print $2'}` echo $POS } check_count() { COUNT=`mysql -h127.0.0.1 --port=$2 test -e "select count(*) as value from t1\G" | grep value | awk '{print $2}'` if [ "$COUNT" = "$3" ]; then return else echo "$1 [Fail (COUNT $COUNT is not equal to expected count $3)]" exit 1 fi } check_relay_purge() { PURGE=`mysql -h127.0.0.1 --port=$2 test -e "select @@global.relay_log_purge\G" | grep global | awk '{print $2}'` if [ "$PURGE" = "$3" ]; then return else echo "$1 [Fail (relay_log_purge $PURGE is not equal to expected value $3)]" exit 1 fi } wait_until_manager_start() { i=1 while : do masterha_check_status --conf=$CONF $2 > /dev/null 2>&1 RC=$? if [ "$RC" = "0" ]; then break fi i=`expr $i + 1` if [ $i -gt 120 ]; then echo "$1 [Fail (master_check_status does not become running within 120 seconds)]" exit 1 fi sleep 1 done } wait_until_count() { i=1 while : do ROW_COUNT=`mysql -h127.0.0.1 --port=$2 test -e "select count(*) from t1\G" 2> /dev/null | grep count | awk {'print $2'}` if [ "$ROW_COUNT" = $3 ]; then break fi i=`expr $i + 1` if [ $i -gt 120 ]; then echo "$1 [Fail (wait timeout)]" exit 1 fi sleep 1 done } mha4mysql-manager-0.58/tests/t/force_start_m.sh000077500000000000000000000001741325502577300216030ustar00rootroot00000000000000rm -f $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data/*.pid $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/start $@ > /dev/null mha4mysql-manager-0.58/tests/t/grant.sql000066400000000000000000000001771325502577300202540ustar00rootroot00000000000000grant all privileges on *.* to '"\'!=#&;`=|*?$\\' identified by 'pass"\'!#&;`|*?~<>^()[]{}$\\, "\'!#&;`|*?~=<>^()[]{}$\\, \\'; mha4mysql-manager-0.58/tests/t/grant_nopass.sql000066400000000000000000000002011325502577300216230ustar00rootroot00000000000000grant all privileges on *.* to 'admin' identified by ''; update mysql.user set password='' where user='admin'; flush privileges; mha4mysql-manager-0.58/tests/t/init.sh000077500000000000000000000032621325502577300177200ustar00rootroot00000000000000#/bin/sh . ./env.sh SANDBOX_HOME_ESC=`echo $SANDBOX_HOME | sed -e 's/\//\\\\\\//g'` for tmpl in `ls mha_test*.cnf.tmpl` do cnf=${tmpl%.tmpl} sed -e "s/##VERSION_DIR##/$VERSION_DIR/g" -e "s/##SANDBOX_HOME##/$SANDBOX_HOME_ESC/g" \ -e "s:##CLIENT_BINDIR##:$CLIENT_BINDIR:" -e "s:##CLIENT_LIBDIR##:$CLIENT_LIBDIR:" \ $tmpl > $cnf done $SANDBOX_HOME/send_kill_all > bootstrap.log 2>&1 $SANDBOX_HOME/clear_all > bootstrap.log 2>&1 export MYSQL_USER="" export MYSQL_PWD="" make_replication_sandbox --how_many_slaves=4 --upper_directory=$SANDBOX_HOME --sandbox_base_port=$MP $VERSION >> bootstrap.log 2>&1 export MYSQL_USER=root export MYSQL_PWD=msandbox if [ "A$INIT_SCRIPT" != "A" ]; then eval $INIT_SCRIPT fi rm -f /var/tmp/*127.0.0.1_*.binlog rm -f /var/tmp/*127.0.0.1_*.log rm -f /var/tmp/mha_test* sleep 1 mysql $M -e "source grant.sql" mysql $M -e "source grant_nopass.sql" mysql $S1 -e "source grant_nopass.sql" mysql $S2 -e "source grant_nopass.sql" mysql $S3 -e "source grant_nopass.sql" mysql $S4 -e "source grant_nopass.sql" mysql $M test -e "create table t1 (id int primary key, value int, value2 text); insert into t1 values(1, 100, 'abc');" wait_until_count $0 $S1P 1 wait_until_count $0 $S2P 1 wait_until_count $0 $S3P 1 wait_until_count $0 $S4P 1 if [ "A$USE_GTID_AUTO_POS" != "A" ]; then mysql $S1 -e "stop slave; change master to master_auto_position=1; start slave" > /dev/null 2>&1 mysql $S2 -e "stop slave; change master to master_auto_position=1; start slave" > /dev/null 2>&1 mysql $S3 -e "stop slave; change master to master_auto_position=1; start slave" > /dev/null 2>&1 mysql $S4 -e "stop slave; change master to master_auto_position=1; start slave" > /dev/null 2>&1 fi mha4mysql-manager-0.58/tests/t/insert.pl000077500000000000000000000012751325502577300202640ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings; use DBI; my ( $port, $user, $pass, $start, $stop, $single_tran, $rollback ) = @ARGV; $rollback = 0 if ( !defined($rollback) ); my $dsn = "DBI:mysql:test;host=127.0.0.1;port=$port"; my $dbh = DBI->connect( $dsn, $user, $pass ); my $sth; if ($single_tran) { $sth = $dbh->prepare("BEGIN"); $sth->execute(); } for ( my $i = $start ; $i <= $stop ; $i++ ) { $sth = $dbh->prepare("INSERT INTO t1 VALUES(?, ?, 'aaaaaaa')"); my @params; push @params, $i; push @params, $i; $sth->execute(@params); } if ($single_tran) { if ($rollback) { $sth = $dbh->prepare("ROLLBACK"); } else { $sth = $dbh->prepare("COMMIT"); } $sth->execute(); } mha4mysql-manager-0.58/tests/t/insert_binary.pl000066400000000000000000000013201325502577300216140ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings FATAL=>'all'; use DBI; my ( $port, $user, $pass ) = @ARGV; my $file= "/tmp/test.bin"; open my $out, ">", $file or die "file open error $!\n"; binmode $out; print $out pack('C*', 65, 66, 67, 68, 69, 13, 10, 68, 69); #print $out pack('C*', 65, 66, 67, 0, 68, 69, 13, 10, 68, 69); close($out); open my $in , "<", $file or die "file open error $!\n"; binmode $in; my $buf; read($in, $buf, -s $file); close($in); unlink $file; my $dsn = "DBI:mysql:test;host=127.0.0.1;port=$port"; my $dbh = DBI->connect( $dsn, $user, $pass ); my $value= $buf; my $query= "INSERT INTO binfile (bin_data) VALUES ('$value')"; my $sth= $dbh->prepare($query); $sth->execute(); $dbh->disconnect(); mha4mysql-manager-0.58/tests/t/kill_m.sh000077500000000000000000000002701325502577300202200ustar00rootroot00000000000000kill -9 `ps -ef | grep rsandbox_$VERSION_DIR | grep master | grep mysqld_safe | awk '{print $2}'` kill -9 `cat $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data/mysql_sandbox10000.pid` mha4mysql-manager-0.58/tests/t/master_ip_failover000077500000000000000000000035571325502577300222250ustar00rootroot00000000000000#!/usr/bin/env perl # Test purpose. Checking arguments only use strict; use warnings FATAL => 'all'; use Getopt::Long; my ( $command, $orig_master_host, $orig_master_ip, $orig_master_port, $orig_master_user, $orig_master_password, $new_master_host, $new_master_ip, $new_master_port, $new_master_user, $new_master_password ); GetOptions( 'command=s' => \$command, 'orig_master_host=s' => \$orig_master_host, 'orig_master_ip=s' => \$orig_master_ip, 'orig_master_port=i' => \$orig_master_port, 'orig_master_user=s' => \$orig_master_user, 'orig_master_password=s' => \$orig_master_password, 'new_master_host=s' => \$new_master_host, 'new_master_ip=s' => \$new_master_ip, 'new_master_port=i' => \$new_master_port, 'new_master_user=s' => \$new_master_user, 'new_master_password=s' => \$new_master_password, ); exit &main(); sub main { if ( $command eq "stop" || $command eq "stopssh" ) { my $exit_code = 1; eval { if($orig_master_user && $orig_master_password) { if($orig_master_user ne "root" || $orig_master_password ne "msandbox") { die "user account was not correctly passed.\n"; }else { print "user account was correctly passed.\n"; } } $exit_code = 0; }; if ($@) { warn "Got Error: $@\n"; exit $exit_code; } exit $exit_code; } elsif ( $command eq "start" ) { my $exit_code = 10; eval { if($new_master_user ne "root" || $new_master_password ne "msandbox") { die "user account was not correctly passed.\n"; }else { print "user account was correctly passed.\n"; } $exit_code = 0; }; if ($@) { warn $@; exit $exit_code; } exit $exit_code; } elsif ( $command eq "status" ) { exit 0; } else { exit 1; } } mha4mysql-manager-0.58/tests/t/master_ip_failover_blank000077500000000000000000000026021325502577300233620ustar00rootroot00000000000000#!/usr/bin/env perl # Test purpose. Checking arguments only use strict; use warnings FATAL => 'all'; use Getopt::Long; my ( $command, $orig_master_host, $orig_master_ip, $orig_master_port, $orig_master_user, $orig_master_password, $new_master_host, $new_master_ip, $new_master_port, $new_master_user, $new_master_password ); GetOptions( 'command=s' => \$command, 'orig_master_host=s' => \$orig_master_host, 'orig_master_ip=s' => \$orig_master_ip, 'orig_master_port=i' => \$orig_master_port, 'orig_master_user=s' => \$orig_master_user, 'orig_master_password=s' => \$orig_master_password, 'new_master_host=s' => \$new_master_host, 'new_master_ip=s' => \$new_master_ip, 'new_master_port=i' => \$new_master_port, 'new_master_user=s' => \$new_master_user, 'new_master_password=s' => \$new_master_password, ); exit &main(); sub main { if ( $command eq "stop" || $command eq "stopssh" ) { my $exit_code = 1; eval { $exit_code = 0; }; if ($@) { warn "Got Error: $@\n"; exit $exit_code; } exit $exit_code; } elsif ( $command eq "start" ) { my $exit_code = 10; eval { $exit_code = 0; }; if ($@) { warn $@; exit $exit_code; } exit $exit_code; } elsif ( $command eq "status" ) { exit 0; } else { exit 1; } } mha4mysql-manager-0.58/tests/t/master_ip_online_change000077500000000000000000000045631325502577300232050ustar00rootroot00000000000000#!/usr/bin/env perl # Test purpose. Checking arguments only use strict; use warnings FATAL => 'all'; use Getopt::Long; my ( $command, $orig_master_is_new_slave, $orig_master_host, $orig_master_ip, $orig_master_port, $orig_master_user, $orig_master_password, $orig_master_ssh_user, $new_master_host, $new_master_ip, $new_master_port, $new_master_user, $new_master_password, $new_master_ssh_user, ); GetOptions( 'command=s' => \$command, 'orig_master_is_new_slave' => \$orig_master_is_new_slave, 'orig_master_host=s' => \$orig_master_host, 'orig_master_ip=s' => \$orig_master_ip, 'orig_master_port=i' => \$orig_master_port, 'orig_master_user=s' => \$orig_master_user, 'orig_master_password=s' => \$orig_master_password, 'orig_master_ssh_user=s' => \$orig_master_ssh_user, 'new_master_host=s' => \$new_master_host, 'new_master_ip=s' => \$new_master_ip, 'new_master_port=i' => \$new_master_port, 'new_master_user=s' => \$new_master_user, 'new_master_password=s' => \$new_master_password, 'new_master_ssh_user=s' => \$new_master_ssh_user, ); exit &main(); sub main { if($orig_master_ssh_user) { print "orig master ssh user: " . $orig_master_ssh_user . "\n"; }else { die; } if($new_master_ssh_user) { print "new master ssh user: " . $new_master_ssh_user . "\n"; }else { die; } if ( $command eq "stop" || $command eq "stopssh" ) { my $exit_code = 1; eval { if($orig_master_user && $orig_master_password) { if($orig_master_user ne "root" || $orig_master_password ne "msandbox") { die "user account was not correctly passed.\n"; }else { print "user account was correctly passed.\n"; } } $exit_code = 0; }; if ($@) { warn "Got Error: $@\n"; exit $exit_code; } exit $exit_code; } elsif ( $command eq "start" ) { my $exit_code = 10; eval { if($new_master_user ne "root" || $new_master_password ne "msandbox") { die "user account was not correctly passed.\n"; }else { print "user account was correctly passed.\n"; } $exit_code = 0; }; if ($@) { warn $@; exit $exit_code; } exit $exit_code; } elsif ( $command eq "status" ) { exit 0; } else { exit 1; } } mha4mysql-manager-0.58/tests/t/mha_test.cnf.tmpl000066400000000000000000000021361325502577300216640ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script=./master_ip_failover master_ip_online_change_script=./master_ip_online_change report_script="" log_level= debug ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/mha_test_binlog.cnf.tmpl000066400000000000000000000021741325502577300232200ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ ignore_fail=1 node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ ignore_fail=1 node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ [binlog2] hostname=127.0.0.1 master_binlog_dir=/tmp/mha_test mha4mysql-manager-0.58/tests/t/mha_test_connect.cnf.tmpl000066400000000000000000000017171325502577300234010ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 ping_type=connect init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_err1.cnf.tmpl000066400000000000000000000014361325502577300226170ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_err2.cnf.tmpl000066400000000000000000000005761325502577300226240ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master mha4mysql-manager-0.58/tests/t/mha_test_gtid_fail1.cnf.tmpl000066400000000000000000000021211325502577300237410ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## log_level = debug ssh_connection_timeout = 20 [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/datax mha4mysql-manager-0.58/tests/t/mha_test_gtid_fail2.cnf.tmpl000066400000000000000000000022711325502577300237500ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## log_level = debug ssh_connection_timeout = 20 [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/datax [binlog2] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/datax2 mha4mysql-manager-0.58/tests/t/mha_test_gtid_ok.cnf.tmpl000066400000000000000000000022671325502577300233710ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## log_level = debug ssh_connection_timeout = 20 [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/datax [binlog2] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data mha4mysql-manager-0.58/tests/t/mha_test_ignore.cnf.tmpl000066400000000000000000000020761325502577300232320ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ ignore_fail=1 node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ ignore_fail=1 node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/mha_test_large.cnf.tmpl000066400000000000000000000021601325502577300230330ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script=./master_ip_failover master_ip_online_change_script=./master_ip_online_change report_script="" log_level= debug check_repl_delay=0 ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/mha_test_latest.cnf.tmpl000066400000000000000000000020661325502577300232420ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 latest_priority=0 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/mha_test_mm.cnf.tmpl000066400000000000000000000010351325502577300223520ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 mha4mysql-manager-0.58/tests/t/mha_test_mm_online.cnf.tmpl000066400000000000000000000011071325502577300237160ustar00rootroot00000000000000[server default] user=root password=msandbox repl_user=rsandbox repl_password=rsandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 mha4mysql-manager-0.58/tests/t/mha_test_multi.cnf.tmpl000066400000000000000000000017201325502577300230740ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=1 multi_tier_slave=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_multi_online.cnf.tmpl000066400000000000000000000017721325502577300244470ustar00rootroot00000000000000[server default] user=root password=msandbox repl_user=rsandbox repl_password=rsandbox ping_interval=1 multi_tier_slave=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_nobinlog.cnf.tmpl000066400000000000000000000017661325502577300235630ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script=./master_ip_failover master_ip_online_change_script=./master_ip_online_change report_script="" log_level= debug ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_nopass.cnf.tmpl000066400000000000000000000021521325502577300232450ustar00rootroot00000000000000[server default] user=admin ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script=./master_ip_failover master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] password= '' hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] user=root password=msandbox hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] user=root password=msandbox hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/mha_test_online.cnf.tmpl000066400000000000000000000017761325502577300232410ustar00rootroot00000000000000[server default] user=root password=msandbox repl_user=rsandbox repl_password=rsandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script=./master_ip_online_change report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_online_pass.cnf.tmpl000066400000000000000000000024041325502577300242540ustar00rootroot00000000000000[server default] # for a user created by grant all privileges on *.* to '"\'!=#&;`=|*?$\\' identified by 'pass"\'!#&;`|*?~<>^()[]{}$\\, "\'!#&;`|*?~=<>^()[]{}$\\, \\'; user=""'!=#&;`=|*?$\" password= "pass"'!#&;`|*?~<>^()[]{}$\, "'!#&;`|*?~=<>^()[]{}$\, \" repl_user=rsandbox repl_password=rsandbox ping_interval=1 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script=./master_ip_failover_blank report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] user=admin password = '' hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] user=admin password= "" hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_pass.cnf.tmpl000066400000000000000000000023701325502577300227120ustar00rootroot00000000000000[server default] # for a user created by grant all privileges on *.* to '"\'!=#&;`=|*?$\\' identified by 'pass"\'!#&;`|*?~<>^()[]{}$\\, "\'!#&;`|*?~=<>^()[]{}$\\, \\'; user=""'!=#&;`=|*?$\" password= "pass"'!#&;`|*?~<>^()[]{}$\, "'!#&;`|*?~=<>^()[]{}$\, \" ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/mha_test_reset.cnf.tmpl000066400000000000000000000017201325502577300230640ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## skip_reset_slave=1 [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 mha4mysql-manager-0.58/tests/t/mha_test_ssh.cnf.tmpl000066400000000000000000000021211325502577300225330ustar00rootroot00000000000000[server default] user=root password=msandbox ping_interval=3 init_conf_load_script="" shutdown_script="" master_ip_failover_script="" master_ip_online_change_script="" report_script="" ##CLIENT_BINDIR## ##CLIENT_LIBDIR## log_level = debug ssh_connection_timeout = 20 [server1] hostname=127.0.0.1 port=10000 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ candidate_master=1 node_label=sandbox master [server2] hostname=127.0.0.1 port=10001 candidate_master=1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node1/data/ node_label=sandbox slave1 [server3] hostname=127.0.0.1 port=10002 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node2/data/ node_label=sandbox slave2 [server4] hostname=127.0.0.1 port=10003 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node3/data/ node_label=sandbox slave3 [server5] hostname=127.0.0.1 port=10004 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/node4/data/ node_label=sandbox slave4 [binlog1] hostname=127.0.0.1 master_binlog_dir=##SANDBOX_HOME##/rsandbox_##VERSION_DIR##/master/data/ mha4mysql-manager-0.58/tests/t/my-row.cnf000066400000000000000000000001141325502577300203310ustar00rootroot00000000000000[mysqld] #old-passwords binlog-format=row innodb-flush-log-at-trx-commit=0 mha4mysql-manager-0.58/tests/t/my.cnf000066400000000000000000000004231325502577300175270ustar00rootroot00000000000000[mysqld] #old-passwords #performance-schema=0 innodb_log_file_size=8M innodb-flush-log-at-trx-commit=0 log-slave-updates gtid-mode=on enforce-gtid-consistency slave-parallel-workers=5 relay_log_info_repository=table master_info_repository=table binlog_checksum=0 general-log mha4mysql-manager-0.58/tests/t/run.sh000077500000000000000000000003131325502577300175530ustar00rootroot00000000000000#!/bin/sh rm -f /var/tmp/*127.0.0.1_*.binlog rm -f /var/tmp/*127.0.0.1_*.log rm -f /var/tmp/mha_test* masterha_manager --conf=$CONF --log_output=manager.log $@ > manager.log 2>&1 RC=$? sleep 10 exit $RC mha4mysql-manager-0.58/tests/t/run_bg.sh000077500000000000000000000003011325502577300202200ustar00rootroot00000000000000#!/bin/sh rm -f /var/tmp/*127.0.0.1_*.binlog rm -f /var/tmp/*127.0.0.1_*.log rm -f /var/tmp/mha_test* masterha_manager --conf=$CONF --log_output=manager.log $@ < /dev/null > manager.log 2>&1 & mha4mysql-manager-0.58/tests/t/run_tests000077500000000000000000000025211325502577300203670ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings FATAL => 'all'; use Getopt::Long; use File::Copy; my $mysql_version; my $use_row_format; my $use_gtid_auto_pos; my $init_script; my $custom_clients = "no"; my $tests; GetOptions( 'mysql_version=s' => \$mysql_version, 'use_row_format' => \$use_row_format, 'use_gtid_auto_pos' => \$use_gtid_auto_pos, 'init_script=s' => \$init_script, 'custom_clients=s' => \$custom_clients, 'tests=s' => \$tests, ); if ($mysql_version) { $ENV{'VERSION'} = $mysql_version; my $version_dir = $mysql_version; $version_dir =~ s/\./_/g; $ENV{'VERSION_DIR'} = $version_dir; } if ($use_row_format) { $ENV{'USE_ROW_FORMAT'} = "1"; } else { $ENV{'USE_ROW_FORMAT'} = ""; } if ($use_gtid_auto_pos) { $ENV{'USE_GTID_AUTO_POS'} = "1"; } else { $ENV{'USE_GTID_AUTO_POS'} = ""; } if ($init_script) { $ENV{'INIT_SCRIPT'} = $init_script; }else { $ENV{'INIT_SCRIPT'} = ""; } unless ($tests) { $tests= "*"; } $ENV{'CUSTOM_CLIENTS'}=$custom_clients; my @test_files = `ls t_$tests.sh`; foreach my $test_file (@test_files) { chomp($test_file); my $rc = system("sh $test_file"); # copy logs for analysys if ($rc) { my @log_files = `ls *.log`; foreach my $log (@log_files) { chomp($log); next if ( $log =~ /^t_/ ); move $log, "$test_file.$mysql_version.$log"; } } } mha4mysql-manager-0.58/tests/t/start_m.sh000077500000000000000000000001001325502577300204120ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/master/start $@ > /dev/null mha4mysql-manager-0.58/tests/t/start_s1.sh000077500000000000000000000000771325502577300205160ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/node1/start $@ > /dev/null mha4mysql-manager-0.58/tests/t/start_s2.sh000077500000000000000000000000771325502577300205170ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/node2/start $@ > /dev/null mha4mysql-manager-0.58/tests/t/start_s4.sh000077500000000000000000000000771325502577300205210ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/node4/start $@ > /dev/null mha4mysql-manager-0.58/tests/t/stop_m.sh000077500000000000000000000000741325502577300202540ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/master/stop > /dev/null mha4mysql-manager-0.58/tests/t/stop_s1.sh000077500000000000000000000000731325502577300203420ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/node1/stop > /dev/null mha4mysql-manager-0.58/tests/t/stop_s2.sh000077500000000000000000000000731325502577300203430ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/node2/stop > /dev/null mha4mysql-manager-0.58/tests/t/stop_s4.sh000077500000000000000000000000731325502577300203450ustar00rootroot00000000000000$SANDBOX_HOME/rsandbox_$VERSION_DIR/node4/stop > /dev/null mha4mysql-manager-0.58/tests/t/t_4tier.sh000077500000000000000000000042531325502577300203300ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 ./stop_s1.sh ./start_s1.sh --log-slave-updates mysql $S1 -e "stop slave io_thread;start slave io_thread" FILE1=`get_binlog_file $S1P` POS1=`get_binlog_position $S1P` mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file=\"$FILE1\", master_log_pos=$POS1" fi mysql $S2 -e "start slave" ./stop_s2.sh ./start_s2.sh --log-slave-updates mysql $S2 -e "stop slave io_thread;start slave io_thread" FILE2=`get_binlog_file $S2P` POS2=`get_binlog_position $S2P` mysql $S3 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S3 -e "change master to master_host='127.0.0.1', master_port=$S2P, master_user='rsandbox', master_password='rsandbox'" else mysql $S3 -e "change master to master_host='127.0.0.1', master_port=$S2P, master_user='rsandbox', master_password='rsandbox', master_log_file=\"$FILE2\", master_log_pos=$POS2" fi mysql $S3 -e "start slave" sleep 5 check_master $0 $S1P $MP check_master $0 $S2P $S1P check_master $0 $S3P $S2P check_master $0 $S4P $MP mysql $M test -e "insert into t1 values(3, 300, 'aaaaaa')" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? mysql $S2 -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_nonzero $0 $? ./kill_m.sh ./run.sh --conf=mha_test_multi.cnf fail_if_nonzero $0 $? check_master $0 $S2P $S1P check_master $0 $S3P $S2P check_master $0 $S4P $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 4 mha4mysql-manager-0.58/tests/t/t_4tier_subm_dead.sh000077500000000000000000000036071325502577300223350ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 ./stop_s1.sh ./start_s1.sh --log-slave-updates mysql $S1 -e "stop slave io_thread;start slave io_thread" mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4" fi mysql $S2 -e "start slave" ./stop_s2.sh ./start_s2.sh --log-slave-updates mysql $S2 -e "stop slave io_thread;start slave io_thread" mysql $S3 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S3 -e "change master to master_host='127.0.0.1', master_port=$S2P, master_user='rsandbox', master_password='rsandbox'" else mysql $S3 -e "change master to master_host='127.0.0.1', master_port=$S2P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4" fi mysql $S3 -e "start slave" sleep 5 check_master $0 $S1P $MP check_master $0 $S2P $S1P check_master $0 $S3P $S2P check_master $0 $S4P $MP mysql $M test -e "insert into t1 values(3, 300, 'aaaaaa')" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./stop_s2.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./kill_m.sh ./run.sh --conf=mha_test_multi.cnf fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_advisory_connect.sh000077500000000000000000000011761325502577300226530ustar00rootroot00000000000000. ./init.sh masterha_manager --conf=mha_test_connect.cnf --log_output=manager.log > manager.log 2>&1 & manager1_pid=$! wait_until_manager_start $0 --conf=mha_test_connect.cnf masterha_manager --conf=mha_test_connect.cnf --log_output=manager.log --skip_ssh_check >> manager.log 2>&1 & manager2_pid=$! sleep 30 ./waitpid $manager1_pid rc1=$? ./waitpid $manager2_pid rc2=$? masterha_stop --conf=mha_test_connect.cnf >> manager.log 2>&1 if [ $rc1 -ne 10 ] && [ $rc2 -ne 10 ]; then echo "$0 [Fail (both stopped)]" exit 1 fi if [ $rc1 -eq 10 ] && [ $rc2 -eq 10 ]; then echo "$0 [Fail (both running)]" exit 1 fi echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_advisory_select.sh000077500000000000000000000003161325502577300224740ustar00rootroot00000000000000. ./init.sh ./run_bg.sh & wait_until_manager_start $0 masterha_manager --conf=$CONF >> manager.log 2>&1 RC=$? sleep 5 masterha_stop --conf=$CONF >> manager.log 2>&1 fail_if_zero $0 $RC echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_apply_many_logs.sh000077500000000000000000000012711325502577300224730ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" mysql $S3 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 2000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? is_gtid_supported if test $? = 1 then mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2001 else mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2001 fi mha4mysql-manager-0.58/tests/t/t_apply_many_logs2.sh000077500000000000000000000017451325502577300225630ustar00rootroot00000000000000. ./init.sh # Needs to apply lots of relay log files # Relay log files switched many times while master binary log was not switched because max_relay_log_size is so small. In this case, each relay log does not contain a Rotate Event. This test case is to check recover works without rotate event. mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_apply_many_logs3.sh000077500000000000000000000015221325502577300225550ustar00rootroot00000000000000. ./init.sh # Needs to apply lots of relay log files # Relay log files switched many times while master binary log was not switched because max_relay_log_size is so small. In this case, each relay log does not contain a Rotate Event. This test case is to check recover works without rotate event. mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" mysql $S4 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_binary.sh000077500000000000000000000004471325502577300205660ustar00rootroot00000000000000# Needs 5.6.3+ client to pass this test case . ./init.sh mysql $M test -e "create table binfile (bin_data blob) charset=latin1" mysql $M test -e "drop table t1" mysql $S2 -e "stop slave" perl insert_binary.pl $MP $MYSQL_USER $MYSQL_PWD ./kill_m.sh ./run.sh fail_if_nonzero $0 $? ./check $0 1 mha4mysql-manager-0.58/tests/t/t_conf.sh000077500000000000000000000004541325502577300202250ustar00rootroot00000000000000. ./init.sh masterha_check_repl --conf=mha_test_err1.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_err1.cnf fail_if_zero $0 $? masterha_check_repl --conf=mha_test_err2.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_err2.cnf fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_data_io_error.sh000077500000000000000000000017651325502577300221170ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values (100, 100, 100)" mysql $S1 test -e "stop slave io_thread" mysql $S2 test -e "stop slave io_thread" mysql $S3 test -e "stop slave io_thread" mysql $S4 test -e "stop slave io_thread" mysql $M test -e "insert into t1 values (101, 100, 100)" masterha_manager --conf=$CONF --log_output=manager.log --skip_ssh_check --ignore_binlog_server_error > manager.log 2>&1 & manager_pid=$! wait_until_manager_start $0 masterha_check_status --conf=$CONF > /dev/null fail_if_nonzero $0 $? mkdir $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data_tmp mv $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data/mysql-bin* $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data_tmp/ ./kill_m.sh ./waitpid $manager_pid 50 rc1=$? rm -rf $SANDBOX_HOME/rsandbox_$VERSION_DIR/master/data_tmp if [ $rc1 -ne 0 ]; then masterha_stop --conf=$CONF >> manager.log 2>&1 echo "$0 [Fail (got error $rc)]" exit 1 fi mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_dual_master_error.sh000077500000000000000000000005071325502577300230100ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave; reset slave;" masterha_check_repl --conf=mha_test_mm.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_mm.cnf fail_if_zero $0 $? masterha_check_repl --conf=mha_test.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test.cnf fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_filter_incorrect.sh000077500000000000000000000007401325502577300226330ustar00rootroot00000000000000. ./init.sh ./stop_s1.sh ./start_s1.sh --binlog-do-db=aaa masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? ./stop_s1.sh ./start_s1.sh --replicate-ignore-db=aaa masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? ./stop_s1.sh ./start_s1.sh --replicate-do-table=aaa masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_ignore_nostart.sh000077500000000000000000000002461325502577300223340ustar00rootroot00000000000000. ./init.sh ./stop_s4.sh ./run.sh --conf=$CONF_IGNORE fail_if_zero $0 $? ./kill_m.sh ./stop_s4.sh ./run.sh --conf=$CONF_IGNORE fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_ignore_recovery1.sh000077500000000000000000000006701325502577300225620ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values (100, 100, 100)" ./run_bg.sh --conf=$CONF_IGNORE wait_until_manager_start $0 --conf=$CONF_IGNORE ./stop_s4.sh ./kill_m.sh sleep 50 masterha_stop --conf=$CONF_IGNORE > /dev/null ./start_s4.sh mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $S4P $MP check_count $0 $S4P 2 ./check $0 3 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P" mha4mysql-manager-0.58/tests/t/t_ignore_recovery2.sh000077500000000000000000000015141325502577300225610ustar00rootroot00000000000000. ./init.sh # Can't recover S3 due to lacking relay logs mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S3 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 ./run_bg.sh --conf=$CONF_IGNORE wait_until_manager_start $0 --conf=$CONF_IGNORE sleep 30 ./kill_m.sh sleep 100 mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" is_gtid_supported if test $? = 1 then ./check $0 3001 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" else check_master $0 $S3P $MP ./check $0 3001 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S4P" fi mha4mysql-manager-0.58/tests/t/t_ignore_recovery3.sh000077500000000000000000000015731325502577300225670ustar00rootroot00000000000000. ./init.sh # Can't recover S3 and S4 due to lacking relay logs mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S3 test -e "stop slave io_thread" mysql $S4 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 ./run_bg.sh --conf=$CONF_IGNORE wait_until_manager_start $0 --conf=$CONF_IGNORE ./kill_m.sh sleep 100 mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" is_gtid_supported if test $? = 1 then ./check $0 3001 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" else check_master $0 $S3P $MP check_master $0 $S4P $MP ./check $0 3001 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P" fi mha4mysql-manager-0.58/tests/t/t_ignore_recovery4.sh000077500000000000000000000017521325502577300225670ustar00rootroot00000000000000. ./init.sh # Can't recover S3 and S4 due to lacking relay logs mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S3 test -e "stop slave io_thread" mysql $S4 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 #mysql $S2 test -e "stop slave io_thread" mysql $M test -e "insert into t1 values(99999, 300, 'bbbaaaaaaa');" ./run_bg.sh --conf=$CONF_IGNORE wait_until_manager_start $0 --conf=$CONF_IGNORE ./kill_m.sh sleep 100 mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" is_gtid_supported if test $? = 1 then ./check $0 3002 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" else check_master $0 $S3P $MP check_master $0 $S4P $MP ./check $0 3002 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P" fi mha4mysql-manager-0.58/tests/t/t_ignore_start.sh000077500000000000000000000002021325502577300217670ustar00rootroot00000000000000. ./init.sh ./kill_m.sh ./stop_s4.sh ./run.sh --conf=$CONF_IGNORE --ignore_fail_on_start fail_if_nonN $0 $? 10 echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_keep_relay_log_purge.sh000077500000000000000000000006751325502577300234700ustar00rootroot00000000000000. ./init.sh mysql $S3 -e "set global relay_log_purge=0" mysql $S4 -e "set global relay_log_purge=0" mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_relay_purge $0 $S1P 1 check_relay_purge $0 $S2P 1 check_relay_purge $0 $S3P 0 check_relay_purge $0 $S4P 0 ./check $0 3 mha4mysql-manager-0.58/tests/t/t_large_data.sh000077500000000000000000000010721325502577300213600ustar00rootroot00000000000000. ./init.sh mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 100001 mha4mysql-manager-0.58/tests/t/t_large_data_bulk.sh000077500000000000000000000016231325502577300223770ustar00rootroot00000000000000. ./init.sh mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" perl bulk_tran_insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1000 mysql $M test -e "update t1 set value = value+100 where id between 1 and 10000" mysql $M test -e "update t1 set value = value+200 where id between 10001 and 20000" mysql $M test -e "update t1 set value = value+300 where id between 20001 and 30000" mysql $M test -e "update t1 set value = value+400 where id between 30001 and 40000" sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 100001 mha4mysql-manager-0.58/tests/t/t_large_data_bulk_slow.sh000077500000000000000000000021301325502577300234350ustar00rootroot00000000000000. ./init.sh # Fast position search should fail mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S1 test -e "set global max_relay_log_size=65536" mysql $S2 test -e "set global max_relay_log_size=65536" mysql $S3 test -e "set global max_relay_log_size=65536" mysql $S4 test -e "set global max_relay_log_size=65536" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" perl bulk_tran_insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1000 mysql $M test -e "update t1 set value = value+100 where id between 1 and 10000" mysql $M test -e "update t1 set value = value+200 where id between 10001 and 20000" mysql $M test -e "update t1 set value = value+300 where id between 20001 and 30000" mysql $M test -e "update t1 set value = value+400 where id between 30001 and 40000" sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 100001 mha4mysql-manager-0.58/tests/t/t_large_data_slow.sh000077500000000000000000000013771325502577300224340ustar00rootroot00000000000000. ./init.sh # Fast position search should fail mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S1 test -e "set global max_relay_log_size=65536" mysql $S2 test -e "set global max_relay_log_size=65536" mysql $S3 test -e "set global max_relay_log_size=65536" mysql $S4 test -e "set global max_relay_log_size=65536" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 100001 mha4mysql-manager-0.58/tests/t/t_large_data_slow2.sh000077500000000000000000000013241325502577300225060ustar00rootroot00000000000000. ./init.sh # new master(S1)'s sql thread is stopped too long time so S1 can't be a new master # It takes very long time on virtual machine mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" # very long transaction perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 1000000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 1000001 mha4mysql-manager-0.58/tests/t/t_large_data_slow3.sh000077500000000000000000000016261325502577300225140ustar00rootroot00000000000000. ./init.sh # new master(S1)'s sql thread is stopped too long time so S1 can't be a new master # It takes very long time on virtual machine mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" mysql $S1 test -e "stop slave sql_thread" mysql $S2 test -e "stop slave sql_thread" mysql $S3 test -e "stop slave sql_thread" mysql $S4 test -e "stop slave sql_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" # very long transaction perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 1000000 0 sleep 1 ./kill_m.sh ./run.sh --conf=mha_test_large.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 1000001 mha4mysql-manager-0.58/tests/t/t_large_data_sql_fail.sh000077500000000000000000000020471325502577300232350ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 # commits every 1000 rows # sql thread stops normally (simulating offline backup job) mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S2 test -e "stop slave sql_thread" mysql $S2 test -e "insert into t1 values (99950, 100, 100)" check_sql_stop $0 $S2P perl tran_insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1000 ./run_bg.sh & wait_until_manager_start $0 check_sql_stop $0 $S2P masterha_check_status --conf=$CONF > /dev/null fail_if_nonzero $0 $? masterha_stop --conf=$CONF > /dev/null check_sql_stop $0 $S2P ./kill_m.sh ./run.sh fail_if_zero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $S2P $MP check_count $0 $S2P 99001 ./check $0 100001 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" mha4mysql-manager-0.58/tests/t/t_large_data_sql_stop.sh000077500000000000000000000015561325502577300233130ustar00rootroot00000000000000. ./init.sh # commits every 1000 rows # sql thread stops normally (simulating offline backup job) mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S2 test -e "stop slave sql_thread" check_sql_stop $0 $S2P perl tran_insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1000 ./run_bg.sh & wait_until_manager_start $0 check_sql_stop $0 $S2P masterha_check_status --conf=$CONF > /dev/null fail_if_nonzero $0 $? masterha_stop --conf=$CONF > /dev/null check_sql_stop $0 $S2P ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 100001 mha4mysql-manager-0.58/tests/t/t_large_data_tran.sh000077500000000000000000000011021325502577300223760ustar00rootroot00000000000000. ./init.sh mysql $S1 test -e "set global relay_log_purge=0" mysql $S2 test -e "set global relay_log_purge=0" mysql $S3 test -e "set global relay_log_purge=0" mysql $S4 test -e "set global relay_log_purge=0" mysql $S3 test -e "flush logs" mysql $S3 test -e "flush logs" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S1 test -e "stop slave io_thread" perl tran_insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 100000 1000 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 100001 mha4mysql-manager-0.58/tests/t/t_latest_recovery1.sh000077500000000000000000000005701325502577300225720ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=$CONF_LATEST fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 5 mha4mysql-manager-0.58/tests/t/t_latest_recovery2.sh000077500000000000000000000010431325502577300225670ustar00rootroot00000000000000. ./init.sh mysql $S4 -e "set global relay_log_purge=0" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $S2 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=$CONF_LATEST fail_if_nonzero $0 $? mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 6 mha4mysql-manager-0.58/tests/t/t_latest_recovery3.sh000077500000000000000000000011161325502577300225710ustar00rootroot00000000000000. ./init.sh mysql $S4 -e "set global relay_log_purge=0" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" sleep 3 mysql $S2 -e "stop slave io_thread" mysql $S3 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=$CONF_LATEST fail_if_nonzero $0 $? mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 6 mha4mysql-manager-0.58/tests/t/t_manual.sh000077500000000000000000000014671325502577300205620ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values (2, 200, 'aaaaa')" masterha_master_switch --master_state=dead --interactive=0 --dead_master_host=127.0.0.1 --dead_master_ip=127.0.0.1 --dead_master_port=$MP --ssh_reachable=1 --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./kill_m.sh ./stop_s1.sh masterha_master_switch --master_state=dead --interactive=0 --dead_master_host=127.0.0.1 --dead_master_ip=127.0.0.1 --dead_master_port=$MP --ssh_reachable=1 --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./start_s1.sh masterha_master_switch --master_state=dead --interactive=0 --dead_master_host=127.0.0.1 --dead_master_ip=127.0.0.1 --dead_master_port=$MP --ssh_reachable=1 --conf=$CONF > /dev/null 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_mm.sh000077500000000000000000000025441325502577300177130ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi sleep 1 masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" mysql $M -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? mysql $S1 -e "set global read_only=0" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" mysql $M -e "set global read_only=0" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_nonzero $0 $? mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_mm_3tier.sh000077500000000000000000000023741325502577300210220ustar00rootroot00000000000000. ./init.sh POS1=`get_binlog_position $S1P` mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=$POS1; start slave" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 mysql $S1 -e "set global read_only=1" POS2=`get_binlog_position $S1P` mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=$POS2" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? ./kill_m.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_nonzero $0 $? check_master $0 $S2P $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_mm_3tier_subm_dead.sh000077500000000000000000000025631325502577300230250ustar00rootroot00000000000000. ./init.sh mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 mysql $S1 -e "set global read_only=1" mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P ./stop_s1.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_zero $0 $? mysql $M -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? ./kill_m.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_mm_normal.sh000077500000000000000000000012761325502577300212640ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "set global read_only=1" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 4 mha4mysql-manager-0.58/tests/t/t_mm_normal_skip_reset.sh000077500000000000000000000020711325502577300235060ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi ./stop_s1.sh ./start_s1.sh --log-slave-updates sleep 5; mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "set global read_only=1" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" sleep 2; ./kill_m.sh ./run.sh --conf=mha_test_reset.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./force_start_m.sh sleep 2; check_sql_stop $0 $S1P mysql $S1 -e "start slave" check_sql_yes $0 $S1P mysql $M -e "set global read_only=0" mysql $M test -e "insert into t1 values(10000004, 300, 'bbbaaaaaaa');" ./check $0 5 "h=127.0.0.1,P=$MP h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" mha4mysql-manager-0.58/tests/t/t_mm_noslaves.sh000077500000000000000000000025111325502577300216170ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi sleep 1 masterha_check_repl --conf=mha_test_mm.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_mm.cnf fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" mysql $M -e "set global read_only=1" masterha_check_repl --conf=mha_test_mm.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_mm.cnf fail_if_zero $0 $? mysql $M -e "set global read_only=0" masterha_check_repl --conf=mha_test_mm.cnf > /dev/null 2>&1 fail_if_nonzero $0 $? mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=mha_test_mm.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 "h=127.0.0.1,P=$S1P" mha4mysql-manager-0.58/tests/t/t_mm_ro_fail.sh000077500000000000000000000013011325502577300213740ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" mysql $M -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_mm_subm_dead.sh000077500000000000000000000021701325502577300217110ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "set global read_only=1" ./stop_s1.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_zero $0 $? mysql $M -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_nonzero $0 $? mysql $M test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 "h=127.0.0.1,P=$MP" mha4mysql-manager-0.58/tests/t/t_mm_subm_dead_many.sh000077500000000000000000000022031325502577300227320ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "set global read_only=1" ./stop_s1.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_zero $0 $? mysql $M -e "set global read_only=1" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? ./stop_s2.sh masterha_check_repl --conf=mha_test_multi.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_multi.cnf fail_if_zero $0 $? mysql $M test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 "h=127.0.0.1,P=$MP" mha4mysql-manager-0.58/tests/t/t_mm_subm_dead_noslave.sh000077500000000000000000000016551325502577300234470ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $S1 -e "set global read_only=1" ./stop_s1.sh masterha_check_repl --conf=mha_test_mm.cnf > /dev/null 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=dead --conf=mha_test_multi.cnf --interactive=0 --dead_master_host=127.0.0.1 --dead_master_port=$MP > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_mm.cnf fail_if_nonzero $0 $? mysql $M test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 "h=127.0.0.1,P=$MP" mha4mysql-manager-0.58/tests/t/t_needsync_1.sh000077500000000000000000000003261325502577300213260ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_needsync_1_nocm.sh000077500000000000000000000006101325502577300223360ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 mysql $S1 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 2 mysql $S1 -e "SET GLOBAL read_only=1" ./kill_m.sh ./run.sh --skip_change_master --skip_disable_read_only fail_if_nonzero $0 $? is_read_only $0 $S1P check_master $0 $S1P $MP check_master $0 $S2P $MP check_master $0 $S3P $MP check_master $0 $S4P $MP ./check $0 2 mha4mysql-manager-0.58/tests/t/t_needsync_1_nopass.sh000077500000000000000000000007671325502577300227220ustar00rootroot00000000000000. ./init.sh export MYSQL_PWD="" masterha_check_repl --conf=mha_test_nopass.cnf > manager.log 2>&1 fail_if_nonzero $0 $? export MYSQL_PWD=msandbox mysql $S1 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" wait_until_count $0 $S2P 2 wait_until_count $0 $S3P 2 wait_until_count $0 $S4P 2 ./kill_m.sh export MYSQL_PWD="" ./run.sh --conf=mha_test_nopass.cnf export MYSQL_PWD=msandbox mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_needsync_1_pass.sh000077500000000000000000000003571325502577300223600ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=mha_test_pass.cnf mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_needsync_1_ssh.sh000077500000000000000000000003561325502577300222060ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=mha_test_ssh.cnf mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_needsync_2.sh000077500000000000000000000005541325502577300213320ustar00rootroot00000000000000. ./init.sh mysql $S4 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_needsync_2_nobinlog.sh000077500000000000000000000006111325502577300232130ustar00rootroot00000000000000. ./init.sh mysql $S4 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh --conf=mha_test_nobinlog.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_needsync_2_pass.sh000077500000000000000000000006051325502577300223550ustar00rootroot00000000000000. ./init.sh mysql $S4 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh --conf=mha_test_pass.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_needsync_2_ssh.sh000077500000000000000000000006041325502577300222030ustar00rootroot00000000000000. ./init.sh mysql $S4 test -e "set global relay_log_purge=0" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh --conf=mha_test_ssh.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_needsync_binlog.sh000077500000000000000000000014751325502577300224460ustar00rootroot00000000000000. ./init.sh perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 mysql $S3 test -e "stop slave io_thread" mysql $S4 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 2700 0 mysql $S1 test -e "stop slave io_thread" mysql $S2 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2701 3000 0 mysql $M test -e "insert into t1 values(99999, 300, 'bbbaaaaaaa');" ./run_bg.sh --conf=$CONF_BINLOG wait_until_manager_start $0 --conf=$CONF_BINLOG ./kill_m.sh mkdir -p /tmp/mha_test rm -rf /tmp/mha_test/* cp -p $MASTER_DATA_DIR/mysql-bin.* /tmp/mha_test/ sleep 20 mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $S3P $S1P check_master $0 $S4P $S1P ./check $0 3002 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" mha4mysql-manager-0.58/tests/t/t_needsync_fail.sh000077500000000000000000000003251325502577300221000ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_needsync_flush.sh000077500000000000000000000004401325502577300223040ustar00rootroot00000000000000. ./init.sh mysql $S3 -e "set global relay_log_purge=0" mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_needsync_flush2.sh000077500000000000000000000007341325502577300223740ustar00rootroot00000000000000. ./init.sh mysql $S2 -e "set global relay_log_purge=0" mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 300, 'ROLLBACK')" mysql $M test -e "insert into t1 values(4, 400, \"Test\nROLLBACK\")" mysql $M test -e "insert into t1 values(5, 500, 'bbbbb')" ./kill_m.sh ./run.sh mysql $S2 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 6 mha4mysql-manager-0.58/tests/t/t_needsync_flush3.sh000077500000000000000000000010741325502577300223730ustar00rootroot00000000000000. ./init.sh mysql $S3 -e "set global relay_log_purge=0" mysql $S1 -e "stop slave io_thread" mysql $M test -e "flush logs" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 300, 'ROLLBACK')" mysql $M test -e "insert into t1 values(4, 400, \"Test\nROLLBACK\")" mysql $M test -e "insert into t1 values(5, 500, 'bbbbb')" mysql $S2 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(6, 500, 'bbbbb')" ./kill_m.sh ./run.sh mysql $S3 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_needsync_flush_slave.sh000077500000000000000000000010151325502577300234750ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" for i in $S2P $S3P $S4P do mysql -h127.0.0.1 -P$i -e "set global relay_log_purge=0" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" done mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" ./kill_m.sh ./run.sh mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_new_master_heavy.sh000077500000000000000000000012101325502577300226270ustar00rootroot00000000000000. ./init.sh masterha_manager --conf=$CONF --log_output=manager.log --skip_ssh_check >> manager.log 2>&1 & manager1_pid=$! wait_until_manager_start $0 mysql $S1 test -e "SELECT *, sleep(200) from t1" > /dev/null 2>&1 & mysql_pid=$! ./stop_m.sh ./waitpid $manager1_pid 50 rc1=$? kill $mysql_pid wait $mysql_pid 2> /dev/null if [ $rc1 -eq 10 ]; then masterha_stop --conf=$CONF >> manager.log 2>&1 echo "$0 [Fail (timeout)]" exit 1 fi if [ $rc1 -ne 0 ]; then masterha_stop --conf=$CONF >> manager.log 2>&1 echo "$0 [Fail (got error $rc)]" exit 1 fi mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_new_master_heavy_wait.sh000077500000000000000000000012571325502577300236660ustar00rootroot00000000000000. ./init.sh masterha_manager --conf=$CONF --log_output=manager.log --skip_ssh_check >> manager.log 2>&1 & manager1_pid=$! wait_until_manager_start $0 mysql $S1 test -e "set global read_only=1" mysql $S1 test -e "SELECT *, sleep(200) from t1" > /dev/null 2>&1 & mysql_pid=$! ./stop_m.sh ./waitpid $manager1_pid 50 rc1=$? kill $mysql_pid wait $mysql_pid 2> /dev/null if [ $rc1 -eq 10 ]; then masterha_stop --conf=$CONF >> manager.log 2>&1 echo "$0 [Pass(wait)]" exit 1 fi if [ $rc1 -ne 0 ]; then masterha_stop --conf=$CONF >> manager.log 2>&1 echo "$0 [Fail (got error $rc)]" exit 1 fi mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_no_relay_log.sh000077500000000000000000000015141325502577300217470ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 # Needs to apply lots of relay log files # Relay log files switched many times while master binary log was not switched because max_relay_log_size is so small. In this case, each relay log does not contain a Rotate Event. This test case is to check recover works without rotate event. mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_zero $0 $? check_master $0 $S1P $MP check_master $0 $S2P $MP check_master $0 $S3P $MP check_master $0 $S4P $MP echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_no_relay_log_gtid.sh000077500000000000000000000017021325502577300227550ustar00rootroot00000000000000. ./init.sh skip_if_not_gtid $0 # With GTID, failover should succeed regardless of relay logs # Needs to apply lots of relay log files # Relay log files switched many times while master binary log was not switched because max_relay_log_size is so small. In this case, each relay log does not contain a Rotate Event. This test case is to check recover works without rotate event. mysql $S1 -e "set global max_relay_log_size=8192" mysql $S2 -e "set global max_relay_log_size=8192" mysql $S3 -e "set global max_relay_log_size=8192" mysql $S4 -e "set global max_relay_log_size=8192" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 2 1000 0 sleep 2 mysql $S1 test -e "stop slave io_thread" perl insert.pl $MP $MYSQL_USER $MYSQL_PWD 1001 3000 0 sleep 1 ./kill_m.sh ./run.sh fail_if_nonzero $0 $? check_master $0 $S2P $S1P check_master $0 $S3P $S1P check_master $0 $S4P $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3001 mha4mysql-manager-0.58/tests/t/t_normal_crash.sh000077500000000000000000000006411325502577300217460ustar00rootroot00000000000000. ./init.sh masterha_secondary_check -s 127.0.0.1 --master_host=127.0.0.1 --master_ip=127.0.0.1 --master_port=10000 > /dev/null fail_if_nonN $0 $? 3 ./kill_m.sh masterha_secondary_check -s 127.0.0.1 --master_host=127.0.0.1 --master_ip=127.0.0.1 --master_port=10000 > /dev/null fail_if_nonzero $0 $? ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_normal_crash_nocm.sh000077500000000000000000000004461325502577300227650ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 mysql $S1 -e "SET GLOBAL read_only=1" ./kill_m.sh ./run.sh --skip_change_master --skip_disable_read_only fail_if_nonzero $0 $? is_read_only $0 $S1P check_master $0 $S1P $MP check_master $0 $S2P $MP check_master $0 $S3P $MP check_master $0 $S4P $MP echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_online_3tier.sh000077500000000000000000000025551325502577300216760ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S1 -e "reset master" mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_multi.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_multi.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_nonzero $0 $? check_master $0 $S2P $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_count $0 $MP 2 ./check $0 3 mha4mysql-manager-0.58/tests/t/t_online_3tier_slave.sh000077500000000000000000000027211325502577300230630ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 is_gtid_supported if test $? = 0 then mysql $S1 -e "reset master" fi mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_multi.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_multi_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? check_master $0 $S2P $S1P check_master $0 $MP $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_count $0 $MP 3 ./check $0 3 mha4mysql-manager-0.58/tests/t/t_online_3tier_slave_keep.sh000077500000000000000000000026111325502577300240650ustar00rootroot00000000000000. ./init.sh mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 ./stop_m.sh ./start_m.sh --log-slave-updates ./stop_s1.sh ./start_s1.sh --log-slave-updates is_gtid_supported if test $? = 0 then mysql $S1 -e "reset master" fi mysql $S1 -e "stop slave io_thread;start slave io_thread" mysql $S2 -e "stop slave io_thread;start slave io_thread" mysql $S3 -e "stop slave io_thread;start slave io_thread" mysql $S4 -e "stop slave io_thread;start slave io_thread" sleep 5 mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P mysql $M test -e "insert into t1 values(3, 300, 'aaaaaa')" masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_mm_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? check_master $0 $S2P $S1P check_master $0 $MP $S1P check_master $0 $S3P $MP check_master $0 $S4P $MP mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_count $0 $MP 4 ./check $0 4 mha4mysql-manager-0.58/tests/t/t_online_busy.sh000077500000000000000000000027401325502577300216260ustar00rootroot00000000000000. ./init.sh masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave --check_only > switch.log 2>&1 fail_if_nonzero $0 $? check_master $0 $S1P $MP mysql $M -e "create database if not exists mysqlslap" sleep 1 mysqlslap --host=127.0.0.1 --port=$S1P --concurrency=15 --query="select sleep(100)" > /dev/null 2>&1 & pid=$! sleep 15 masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 rc=$? kill $pid wait $pid 2> /dev/null fail_if_zero $0 $rc sleep 10 mysql $M test -e "lock tables t1 write; select sleep(100)" > /dev/null 2>&1 & pid2=$! sleep 1 mysql $M test -e "lock tables t1 write" > /dev/null 2>&1 & pid3=$! sleep 2 masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave --flush_tables=0 > switch.log 2>&1 rc=$? kill $pid2 wait $pid2 2> /dev/null kill $pid3 wait $pid3 2> /dev/null fail_if_zero $0 $rc sleep 5 masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $MP $S1P check_count $0 $MP 2 ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_filter.sh000077500000000000000000000031241325502577300221260ustar00rootroot00000000000000. ./init.sh sleep 1 ./stop_m.sh ./start_m.sh --replicate-do-table=test.aaa mysql $S1 -e "stop slave io_thread;start slave io_thread" mysql $S2 -e "stop slave io_thread;start slave io_thread" mysql $S3 -e "stop slave io_thread;start slave io_thread" mysql $S4 -e "stop slave io_thread;start slave io_thread" sleep 5 masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_zero $0 $? mysql $M -e "show slave status\G" > work.log fail_if_nonempty $0 work.log rm work.log mysql $M -e "change master to master_host='xxxx'" masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave >> switch.log 2>&1 fail_if_zero $0 $? mysql $M -e "show slave status\G" > work.log fail_if_empty $0 work.log rm work.log mysql $M -e "RESET SLAVE /*!50516 ALL */; CHANGE MASTER TO master_host=''" > /dev/null 2>&1 mysql $M -e "show slave status\G" > work.log fail_if_nonempty $0 work.log rm work.log sleep 1 ./stop_m.sh ./start_m.sh mysql $S1 -e "stop slave io_thread;start slave io_thread" mysql $S2 -e "stop slave io_thread;start slave io_thread" mysql $S3 -e "stop slave io_thread;start slave io_thread" mysql $S4 -e "stop slave io_thread;start slave io_thread" sleep 5 masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P >> switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_mm.sh000077500000000000000000000021171325502577300212530ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? mysql $S1 -e "set global read_only=1" mysql $M -e "set global read_only=1" masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? mysql $M -e "set global read_only=0" masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_mm_3tier.sh000077500000000000000000000030131325502577300223550ustar00rootroot00000000000000. ./init.sh POS1=`get_binlog_position $S1P` is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=$POS1; start slave" fi mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 mysql $S1 -e "set global read_only=1" POS2=`get_binlog_position $S1P` mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=$POS2" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_multi.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_nonzero $0 $? check_master $0 $S2P $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" # orig master is already a slave of the new master check_count $0 $MP 3 ./check $0 3 mha4mysql-manager-0.58/tests/t/t_online_mm_3tier_slave.sh000077500000000000000000000030511325502577300235510ustar00rootroot00000000000000. ./init.sh POS1=`get_binlog_position $S1P` is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=$POS1; start slave" fi mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" sleep 1 POS2=`get_binlog_position $S1P` mysql $S1 -e "set global read_only=1" mysql $S2 -e "stop slave" is_gtid_supported if test $? = 1 then mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox'" else mysql $S2 -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=$POS2" fi mysql $S2 -e "start slave" check_master $0 $S2P $S1P masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_multi.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? check_master $0 $S2P $S1P check_master $0 $MP $S1P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_count $0 $MP 3 ./check $0 3 mha4mysql-manager-0.58/tests/t/t_online_mm_skip_reset.sh000077500000000000000000000021101325502577300234740ustar00rootroot00000000000000. ./init.sh is_gtid_supported if test $? = 1 then mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_auto_position=1; start slave" else mysql $M -e "change master to master_host='127.0.0.1', master_port=$S1P, master_user='rsandbox', master_password='rsandbox', master_log_file='mysql-bin.000001', master_log_pos=4; start slave" fi ./stop_s1.sh ./start_s1.sh --log-slave-updates sleep 5; mysql $M -e "set global read_only=0" mysql $S1 -e "set global read_only=1" masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_reset.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_sql_stop $0 $S1P mysql $S1 -e "start slave" check_sql_yes $0 $S1P mysql $M -e "set global read_only=0" mysql $M test -e "insert into t1 values(10000004, 300, 'bbbaaaaaaa');" ./check $0 3 "h=127.0.0.1,P=$MP h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" mha4mysql-manager-0.58/tests/t/t_online_normal.sh000077500000000000000000000011421325502577300221270ustar00rootroot00000000000000. ./init.sh masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_err1.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_err2.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_slave.sh000077500000000000000000000005361325502577300217570ustar00rootroot00000000000000. ./init.sh masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_online.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $MP $S1P check_count $0 $MP 2 ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_slave_fail.sh000077500000000000000000000004641325502577300227520ustar00rootroot00000000000000. ./init.sh masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_zero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_count $0 $MP 1 ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_slave_pass.sh000077500000000000000000000006201325502577300227770ustar00rootroot00000000000000. ./init.sh export MYSQL_PWD="" masterha_master_switch --master_state=alive --interactive=0 --conf=mha_test_online_pass.cnf --new_master_host=127.0.0.1 --new_master_port=$S1P --orig_master_is_new_slave > switch.log 2>&1 fail_if_nonzero $0 $? export MYSQL_PWD=msandbox mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $MP $S1P check_count $0 $MP 2 ./check $0 2 mha4mysql-manager-0.58/tests/t/t_online_slave_sql_stop.sh000077500000000000000000000010551325502577300237000ustar00rootroot00000000000000. ./init.sh mysql $S2 test -e "stop slave sql_thread" check_sql_stop $0 $S2P masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_zero $0 $? mysql $S2 test -e "start slave sql_thread" check_sql_yes $0 $S2P masterha_master_switch --master_state=alive --interactive=0 --conf=$CONF --new_master_host=127.0.0.1 --new_master_port=$S1P > switch.log 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/t_recover_master_fail.sh000077500000000000000000000012351325502577300233110ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $S3 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" mysql $S1 test -e "insert into t1 values(4, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_zero $0 $? #check_master $0 $S1P $MP check_master $0 $S2P $MP check_master $0 $S3P $MP check_master $0 $S4P $MP check_count $0 $S1P 4 check_count $0 $S2P 6 check_count $0 $S3P 1 echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_recover_slave_fail.sh000077500000000000000000000012221325502577300231240ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 mysql $S3 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" mysql $S3 test -e "insert into t1 values(4, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_zero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $S3P $MP check_count $0 $S3P 4 ./check $0 7 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P h=127.0.0.1,P=$S4P" mha4mysql-manager-0.58/tests/t/t_recover_slave_fail2.sh000077500000000000000000000014171325502577300232140ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 mysql $S3 -e "stop slave io_thread" mysql $S4 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" mysql $S3 test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $S4 test -e "insert into t1 values(5, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_zero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" check_master $0 $S3P $MP check_master $0 $S4P $MP check_count $0 $S3P 4 check_count $0 $S4P 5 ./check $0 7 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S2P" mha4mysql-manager-0.58/tests/t/t_recover_slave_ok.sh000077500000000000000000000007771325502577300226400ustar00rootroot00000000000000. ./init.sh mysql $S3 -e "stop slave io_thread" mysql $S4 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_save_binlog_gtid.sh000077500000000000000000000024761325502577300226050ustar00rootroot00000000000000. ./init.sh skip_if_not_gtid $0 mysql $S1 -e "stop slave io_thread" mysql $S2 -e "stop slave io_thread" mysql $S3 -e "stop slave io_thread" mysql $S4 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=mha_test_gtid_fail1.cnf fail_if_zero $0 $? rm -f /var/tmp/mha_test* masterha_master_switch --master_state=dead --interactive=0 --dead_master_host=127.0.0.1 --dead_master_ip=127.0.0.1 --dead_master_port=$MP --conf=mha_test_gtid_fail1.cnf > /dev/null 2>&1 fail_if_zero $0 $? rm -f /var/tmp/mha_test* masterha_master_switch --master_state=dead --interactive=0 --dead_master_host=127.0.0.1 --dead_master_ip=127.0.0.1 --dead_master_port=$MP --conf=mha_test_gtid_fail2.cnf > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh --conf=mha_test_gtid_ok.cnf masterha_master_switch --master_state=dead --interactive=0 --dead_master_host=127.0.0.1 --dead_master_ip=127.0.0.1 --dead_master_port=$MP --conf=mha_test_gtid_ok.cnf > /dev/null 2>&1 fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_save_master_log.sh000077500000000000000000000011071325502577300224460ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $S2 -e "stop slave io_thread" mysql $S3 -e "stop slave io_thread" mysql $S4 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_save_master_log_pass.sh000077500000000000000000000011401325502577300234710ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $S2 -e "stop slave io_thread" mysql $S3 -e "stop slave io_thread" mysql $S4 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=mha_test_pass.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_save_master_log_ssh.sh000077500000000000000000000011371325502577300233260ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" mysql $S2 -e "stop slave io_thread" mysql $S3 -e "stop slave io_thread" mysql $S4 -e "stop slave io_thread" mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(3, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(4, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(5, 200, 'aaaaaa')" mysql $M test -e "insert into t1 values(6, 200, 'aaaaaa')" ./kill_m.sh ./run.sh --conf=mha_test_ssh.cnf fail_if_nonzero $0 $? mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 7 mha4mysql-manager-0.58/tests/t/t_slave_incorrect.sh000077500000000000000000000014601325502577300224600ustar00rootroot00000000000000. ./init.sh mysql $S1 -e "stop slave io_thread" for i in $S2P $S3P $S4P do mysql -h127.0.0.1 -P$i -e "set global relay_log_purge=0" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" mysql -h127.0.0.1 -P$i test -e "flush logs" done mysql $M test -e "insert into t1 values(2, 200, 'aaaaaa')" is_gtid_supported if test $? = 1 then mysql $S3 -e "stop slave; reset slave all" else mysql $S3 -e "stop slave; reset slave; change master to master_log_file=''" fi masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? ./kill_m.sh masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? echo "$0 [Pass]" mha4mysql-manager-0.58/tests/t/t_slave_sql_start.sh000077500000000000000000000007161325502577300225070ustar00rootroot00000000000000. ./init.sh mysql $S2 -e "stop slave sql_thread" mysql $M test -e "insert into t1 values (100, 100, 100)" check_sql_stop $0 $S2P ./run_bg.sh & wait_until_manager_start $0 check_sql_stop $0 $S2P masterha_check_status --conf=$CONF > /dev/null fail_if_nonzero $0 $? masterha_stop --conf=$CONF > /dev/null check_sql_stop $0 $S2P ./kill_m.sh ./run.sh check_sql_yes $0 $S2P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 mha4mysql-manager-0.58/tests/t/t_slave_sql_start2.sh000077500000000000000000000014241325502577300225660ustar00rootroot00000000000000. ./init.sh skip_if_gtid $0 mysql $M test -e "insert into t1 values (10, 100, 100)" mysql $M test -e "insert into t1 values (11, 100, 100)" # S1 SQL thread stops with error mysql $S1 test -e "insert into t1 values (100, 100, 100)" mysql $M test -e "insert into t1 values (100, 100, 100)" sleep 1 check_sql_stop $0 $S1P masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? ./run.sh fail_if_zero $0 $? mysql $S1 test -e "delete from t1 where id=100" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_zero $0 $? mysql $S1 test -e "start slave sql_thread" masterha_check_repl --conf=$CONF > /dev/null 2>&1 fail_if_nonzero $0 $? ./kill_m.sh ./run.sh check_sql_yes $0 $S2P mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 5 mha4mysql-manager-0.58/tests/t/t_slave_sql_start3.sh000077500000000000000000000010361325502577300225660ustar00rootroot00000000000000. ./init.sh mysql $S2 test -e "stop slave sql_thread" mysql $S2 test -e "insert into t1 values (100, 100, 100)" mysql $M test -e "insert into t1 values (100, 100, 100)" check_sql_stop $0 $S2P ./run_bg.sh & wait_until_manager_start $0 check_sql_stop $0 $S2P masterha_check_status --conf=$CONF > /dev/null masterha_stop --conf=$CONF > /dev/null check_sql_stop $0 $S2P ./kill_m.sh ./run.sh mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 3 "h=127.0.0.1,P=$S1P h=127.0.0.1,P=$S3P h=127.0.0.1,P=$S4P" mha4mysql-manager-0.58/tests/t/t_slave_stop.sh000077500000000000000000000002571325502577300214600ustar00rootroot00000000000000. ./init.sh ./stop_s1.sh ./run.sh fail_if_zero $0 $? ./start_s1.sh ./kill_m.sh ./run.sh mysql $S1 test -e "insert into t1 values(10000003, 300, 'bbbaaaaaaa');" ./check $0 2 mha4mysql-manager-0.58/tests/t/tran_insert.pl000077500000000000000000000016231325502577300213050ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings; use DBI; my ( $port, $user, $pass, $start, $stop, $commit_interval ) = @ARGV; my $pk = $start; my $committed_rows = 0; my $dsn = "DBI:mysql:test;host=127.0.0.1;port=$port"; my $dbh = DBI->connect( $dsn, $user, $pass ); if ( $commit_interval == 1 ) { $dbh->do("SET AUTOCOMMIT=1"); } else { $dbh->do("SET AUTOCOMMIT=0"); } while ( $pk <= $stop ) { my ( $sth, $ret ); $sth = $dbh->prepare(<<'SQL'); insert into t1 values(?, ?, ?) SQL eval { $ret = $sth->execute( $pk, $pk, 'aaaaaa' . $pk ); die if ( $ret != 1 ); }; if ($@) { return 1; } if ( $commit_interval == 1 ) { $committed_rows++; } elsif ( $commit_interval > 1 && $pk % $commit_interval == 0 ) { eval { $dbh->do("COMMIT"); }; if ($@) { return 1; } $committed_rows += $commit_interval; } $pk++; } mha4mysql-manager-0.58/tests/t/waitpid000077500000000000000000000007411325502577300200040ustar00rootroot00000000000000#!/usr/bin/perl use strict; use warnings FATAL=>'all'; my $pid= $ARGV[0]; my $timeout= $ARGV[1]; $timeout= 5 unless($timeout); my $exit_code= 1; eval { local $SIG{ALRM} = sub { die "timeout"; }; alarm $timeout; while(1) { my $pid_exists= kill 0, $pid; if ($pid_exists) { sleep 1; } else { last; } } alarm 0; $exit_code= 0; }; alarm 0; if ($@) { $exit_code= 1; if ($@ =~ m/timeout/) { $exit_code= 10; } } exit $exit_code;