Merge lp:~mysql-mmm-core/mysql-mmm/devel into lp:mysql-mmm

[Pascal Hofmann]

  * Added manual mode (bug #531011), wait mode, config values 'mode' and 'wait_for_other_master'
  * Don't die at startup when no network connection is available - wait for it to appear instead (bug #416572)
  * Changed startup behaviour. mmm_mond will only go into passive mode if it detects the active_master_role on more than one host.
  * Added config value 'careful_startup' (bug #422549). If set to 0 mmm_mond won't ever switch into passive mode at startup.
  * Added check for invalid agent commands (prevents crash when mmmd_mon version 1.x talks to an 2.x agent).

[Pascal Hofmann (pascalhofmann)]

review: Approve

=== modified file 'doc/mmm_configuration.texi'
--- doc/mmm_configuration.texi	2010-02-05 17:27:32 +0000
+++ doc/mmm_configuration.texi	2010-03-09 10:25:26 +0000
@@ -231,6 +231,29 @@
 @item Used by:			@tab monitor
 @end multitable
 
+@item @strong{careful_startup}
+@multitable @columnfractions 0.2 0.8
+@item Description:		@tab Startup carefully i.e. switch into passive mode when writer role is configured on multiple hosts.
+@item Allowed values:	@tab true/yes/1/on false/no/0/off
+@item Default value:	@tab 0
+@item Used by:			@tab monitor
+@end multitable
+
+@item @strong{mode}
+@multitable @columnfractions 0.2 0.8
+@item Description:		@tab Default mode of monitor.
+@item Allowed values:	@tab active manual wait passive
+@item Default value:	@tab active
+@item Used by:			@tab monitor
+@end multitable
+
+@item @strong{wait_for_other_master}
+@multitable @columnfractions 0.2 0.8
+@item Description:		@tab How many seconds to wait for other master to become @code{ONLINE} before switching from mode @code{WAIT} to mode @code{ACTIVE}. 0 = infinite.
+@item Default value:	@tab 120
+@item Used by:			@tab monitor
+@end multitable
+
 @end itemize
 
 

=== modified file 'doc/mmm_control.texi'
--- doc/mmm_control.texi	2010-02-18 14:32:50 +0000
+++ doc/mmm_control.texi	2010-03-09 10:25:26 +0000
@@ -132,7 +132,7 @@
 @end example
 
 @noindent
-See @ref{Passive mode}.
+See @ref{Modes}.
 
 @section @code{set_active}
 Switch the monitor into @code{ACTIVE} mode:
@@ -143,7 +143,18 @@
 @end example
 
 @noindent
-See @ref{Passive mode}.
+See @ref{Modes}.
+
+@section @code{set_manual}
+Switch the monitor into @code{MANUAL} mode:
+
+@example
+# mmm_control set_manual
+OK: Switched into manual mode.
+@end example
+
+@noindent
+See @ref{Modes}.
 
 @section @code{set_passive}
 Switch the monitor into @code{PASSIVE} mode:
@@ -154,10 +165,10 @@
 @end example
 
 @noindent
-See @ref{Passive mode}.
+See @ref{Modes}.
 
 @section @code{move_role @var{role} @var{host}}
-Used to move an exclusive role between the cluster nodes. This command is available in @code{ACTIVE} mode only. Lets assume the following situation:
+Used to move an exclusive role between the cluster nodes. This command is not available in @code{PASSIVE} mode. Lets assume the following situation:
 
 @smallexample
 # mmm_control show
@@ -179,96 +190,8 @@
 @end smallexample
 
 @section @code{move_role --force @var{role} @var{host}}
-Can be used to move the @var{active_master_role} to a host with state @code{REPLICATION_FAIL} or @code{REPLICATION_BACKLOG}. Use this with caution! This command is available in @code{ACTIVE} mode only.
+Can be used to move the @var{active_master_role} to a host with state @code{REPLICATION_FAIL} or @code{REPLICATION_BACKLOG}. Use this with caution! This command is not available in @code{PASSIVE} mode.
 
 @section @code{set_ip @var{ip} @var{host}}
-@code{set_ip} can be used to manipulate the roles in @code{PASSIVE} mode. The changes won't be applied until the monitor is switched into @code{ACTIVE} mode via @code{set_active}.
-
-@*
-Let's assume we have our cluster up and running with the following status:
-
-@smallexample
-# mmm_control show
-  db1(192.168.0.31) master/ONLINE. Roles: writer(192.168.0.50)
-  db2(192.168.0.32) master/ONLINE. Roles: reader(192.168.0.51)
-  db3(192.168.0.33) slave/ONLINE. Roles: reader(192.168.0.52), reader(192.168.0.53)
-@end smallexample
-
-@noindent
-Now, several bad things happen:
-@enumerate
-@item network connection to db1 fails
-@item mmm_mond detects that db1 has failed
-@item mmm_mond moves the writer role to db2, but can't remove it from db1 (because it can't connect to it)
-@item mmm_mond crashes and the status file gets corrupted.
-@item network connection to db1 recovers
-@item The admin restarts mmm_mond
-@end enumerate
-
-mmm_mond has no status information now, and two nodes report, that they have the 
-@code{writer} role, so mmm_mond doesn't know what it should do and will switch 
-into @code{PASSIVE} mode.
-
-@smallexample
-# mmm_control mode
-PASSIVE
-
-# mmm_control show
-# --- Monitor is in PASSIVE MODE ---
-# Cause: Discrepancies between stored status, agent status and system status during startup.
-#
-# Stored status:
-#   db1(192.168.0.31) master/UNKNOWN. Roles:
-#   db2(192.168.0.32) master/UNKNOWN. Roles:
-#   db3(192.168.0.33) slave/UNKNOWN. Roles:
-#
-# Agent status:
-#   db1 ONLINE. Roles: writer(192.168.0.50). Master: ?
-#   db2 ONLINE. Roles: writer(192.168.0.50), reader(192.168.0.51). Master: ?
-#   db3 ONLINE. Roles: reader(192.168.0.52), reader(192.168.0.53). Master: db2
-#
-# System status:
-#   db1 writable. Roles: writer(192.168.0.50)
-#   db2 writable. Roles: writer(192.168.0.50), reader(192.168.0.51)
-#   db3 readonly. Roles: reader(192.168.0.52), reader(192.168.0.53)
-#
-  db1(192.168.0.31) master/ONLINE. Roles: writer(192.168.0.50)
-  db2(192.168.0.32) master/ONLINE. Roles: reader(192.168.0.51)
-  db3(192.168.0.33) slave/ONLINE. Roles: reader(192.168.0.52), reader(192.168.0.53)
-@end smallexample
-
-@noindent
-As you see, mmm_mond tries to recover the status as well as possible. But in this situation it's wrong so one must move the writer role to db2 manually: 
-
-@smallexample
-# mmm_control set_ip 192.168.0.50 db2
-OK: Set role 'writer(192.168.0.50)' to host 'db2'.
-@end smallexample
-
-@noindent
-Now take a look at the status, everything looks ok:
-
-@smallexample
-# mmm_control show
-# --- Monitor is in PASSIVE MODE ---
-# [...]
-  db1(192.168.0.31) master/ONLINE. Roles:
-  db2(192.168.0.32) master/ONLINE. Roles: writer(192.168.0.50), reader(192.168.0.51)
-  db3(192.168.0.33) slave/ONLINE. Roles: reader(192.168.0.52), reader(192.168.0.53)
-@end smallexample
-
-@noindent
-Finally switch the monitor into active mode, so that it will apply the roles:
-
-@smallexample
-# mmm_control set_active
-OK: Switched into active mode.
-
-# mmm_control show
-  db1(192.168.0.31) master/ONLINE. Roles: reader(192.168.0.51)
-  db2(192.168.0.32) master/ONLINE. Roles: writer(192.168.0.50)
-  db3(192.168.0.33) slave/ONLINE. Roles: reader(192.168.0.52), reader(192.168.0.53)
-@end smallexample
-
-@*
-@strong{Note:} The role @code{reader(192.168.0.51)} has been moved to db1, because @code{reader} is a @code{balanced} role.
+@code{set_ip} can be used to manipulate the roles in @code{PASSIVE} mode. The changes won't be applied until the monitor is switched into @code{ACTIVE} or @code{manual} mode via @code{set_active} or @code{set_manual}.
+

=== modified file 'doc/mmm_monitor.texi'
--- doc/mmm_monitor.texi	2010-02-05 17:27:32 +0000
+++ doc/mmm_monitor.texi	2010-03-09 10:25:26 +0000
@@ -108,7 +108,7 @@
 @end itemize
 
 @noindent
-If the network connection doesn't work during startup, mmm_mond will switch into passive mode (@pxref{Passive mode}).
+If the network connection doesn't work during startup, mmm_mond will delay startup until it's available again.
 
 
 @node Flapping
@@ -130,20 +130,22 @@
 If @var{auto_set_online} is > 0, flapping hosts will automatically be set to @code{ONLINE} 
 after @var{flap_duration} seconds.
 
-@node Passive mode
-@section Passive mode
-@cindex passive mode
-
-entered if no network connection during startup
-entered if discrepancies are detected during startup
-entered with set_passive
-
-roles can be changed (unclean) with set_ip
-changed to active with set_active
-
-roles get never changed automatically
-nothing is send to agents
-status file won't be updated
+@node Modes
+@section Modes
+@cindex Modes
+
+@subsection Active mode
+The monitor will remove roles from failed hosts and assign them to other hosts automatically.
+@subsection Manual mode
+The monitor will only distribute @code{balanced} roles across the hosts, but will not remove roles from failed hosts automatically. You can remove roles from failed hosts manually with @code{move_role}.
+@subsection Wait mode
+Like @code{MANUAL} mode, but mode will be changed into @code{ACTIVE} mode when both master hosts are @code{ONLINE} or after @code{wait_for_other_master} seconds have elapsed.
+@subsection Passive mode
+In passive mode the monitor doesn't change roles, update the status file nor send anything to agents.
+In passive mode you can modify roles (unclean) with @code{set_ip} - the changes won't be applied until the monitor is switched to @code{ACTIVE} or @code{MANUAL} mode with @code{set_active} or @code{set_manual}.
+Passive mode will be entered if conflicting roles are detected during startup. You should then analyze the situation, fix the role information (if needed) and switch into @code{ACTIVE} or @code{MANUAL} mode.
+It also can be entered manually with @code{set_passive}.
+
 
 @node Startup
 @section Startup
@@ -152,6 +154,7 @@
 @itemize
 
 @item Initial network check
+@item If network is down startup will be delayed until it's reachable again.
 @item Initial host checks
 @item reads status information from ...
 @itemize @minus
@@ -159,7 +162,7 @@
 @item agents (agent info)
 @item hosts (system info)
 @end itemize
-@item If status information doesn't match or network is down @code{PASSIVE} mode will be entered.
+and tries to figure out the cluster status.
 @end itemize
 
 @node Role transition

=== modified file 'lib/Agent/Agent.pm'
--- lib/Agent/Agent.pm	2009-10-30 07:19:35 +0000
+++ lib/Agent/Agent.pm	2010-03-09 10:25:26 +0000
@@ -4,7 +4,9 @@
 use warnings FATAL => 'all';
 use English qw(EVAL_ERROR);
 use Algorithm::Diff;
+use DBI;
 use Class::Struct;
+use Errno qw(EINTR);
 use Log::Log4perl qw(:easy);
 use MMM::Common::Role;
 use MMM::Common::Socket;
@@ -81,6 +83,7 @@
 	DEBUG "Received Command $cmd";
 	my ($cmd_name, $version, $host, @params) = split('\|', $cmd, -1);
 
+	return "ERROR: Invalid command '$cmd'!" unless (defined($host));
 	return "ERROR: Invalid hostname in command ($host)! My name is '" . $self->name . "'" if ($host ne $self->name);
 	
 	if ($version > main::MMM_PROTOCOL_VERSION) {
@@ -114,7 +117,23 @@
 sub cmd_get_system_status($) {
 	my $self	= shift;
 
-	# TODO maybe determine and send master info if we are a slave host.
+	# determine master info
+    my $dsn			= sprintf("DBI:mysql:host=%s;port=%s;mysql_connect_timeout=3", $self->ip, $self->mysql_port);
+    my $eintr		= EINTR;
+	my $master_ip	= '';
+
+    my $dbh;
+CONNECT: {
+    DEBUG "Connecting to mysql";
+    $dbh   = DBI->connect($dsn, $self->mysql_user, $self->mysql_password, { PrintError => 0 });
+    unless ($dbh) {
+        redo CONNECT if ($DBI::err == 2003 && $DBI::errstr =~ /\($eintr\)/);
+        WARN "Couldn't connect to mysql. Can't determine current master host." . $DBI::err . " " . $DBI::errstr;
+    }
+}
+
+    my $slave_status = $dbh->selectrow_hashref('SHOW SLAVE STATUS');
+	$master_ip = $slave_status->{Master_Host} if (defined($slave_status));
 
 	my @roles;
 	foreach my $role (keys(%{$main::config->{role}})) {
@@ -133,7 +152,7 @@
 	return "ERROR: Could not check if MySQL is writable: $res" if ($ret == 255);
 	my $writable = ($ret == 1);
 
-	my $answer = join('|', ($writable, join(',', @roles)));
+	my $answer = join('|', ($writable, join(',', @roles), $master_ip));
 	return "OK: Returning status!|$answer";
 }
 

=== modified file 'lib/Common/Config.pm'
--- lib/Common/Config.pm	2010-02-03 09:06:11 +0000
+++ lib/Common/Config.pm	2010-03-09 10:25:26 +0000
@@ -37,7 +37,10 @@
 		'flap_duration'			=> { 'default' => 60 * 60 },
 		'flap_count'			=> { 'default' => 3 },
 		'auto_set_online'		=> { 'default' => 0 },
-		'kill_host_bin'			=> { 'default' => 'kill_host' }
+		'kill_host_bin'			=> { 'default' => 'kill_host' },
+		'careful_startup'		=> { 'default' => 1, 'boolean' => 1 },
+		'mode'					=> { 'default' => 'active', 'values' => ['passive', 'active', 'manual', 'wait'] },
+		'wait_for_other_master'	=> { 'default' => 120 }
 		}
 	},
 	'socket'				=> { 'create_if_empty' => ['AGENT', 'CONTROL', 'MONITOR'], 'section' => {

=== modified file 'lib/Monitor/Checker.pm'
--- lib/Monitor/Checker.pm	2010-02-08 15:06:09 +0000
+++ lib/Monitor/Checker.pm	2010-03-09 10:25:26 +0000
@@ -184,7 +184,7 @@
 	my $self	= shift;
 	my $name	= $self->{name};
 
-	DEBUG "Pinging checker '$name'...";
+#	DEBUG "Pinging checker '$name'...";
 
 	my $reader = $self->{reader};
 	my $writer = $self->{writer};
@@ -202,7 +202,7 @@
 		return 0;
 	}
 
-	DEBUG "Checker '$name' is OK ($recv_res)";
+#	DEBUG "Checker '$name' is OK ($recv_res)";
 	return 1;
 }
 

=== modified file 'lib/Monitor/Commands.pm'
--- lib/Monitor/Commands.pm	2010-03-03 00:34:21 +0000
+++ lib/Monitor/Commands.pm	2010-03-09 10:25:26 +0000
@@ -61,7 +61,7 @@
 	my $roles	= MMM::Monitor::Roles->instance();
 
 	my $ret = '';
-	if ($monitor->passive) {
+	if ($monitor->is_passive) {
 		$ret .= "--- Monitor is in PASSIVE MODE ---\n";
 		$ret .= sprintf("Cause: %s\n", $monitor->passive_info);
 		$ret =~ s/^/# /mg;
@@ -193,7 +193,7 @@
 
 	FATAL "Admin changed state of '$host' from $host_state to ADMIN_OFFLINE";
 	$agents->set_state($host, 'ADMIN_OFFLINE');
-	MMM::Monitor::Roles->instance()->clear_host_roles($host);
+	MMM::Monitor::Roles->instance()->clear_roles($host);
 	MMM::Monitor::Monitor->instance()->send_agent_status($host);
 
     return "OK: State of '$host' changed to ADMIN_OFFLINE. Now you can wait some time and check all roles!";
@@ -203,7 +203,7 @@
 	my $ip		= shift;
 	my $host	= shift;
 
-	return "ERROR: This command is only allowed in passive mode" unless (MMM::Monitor::Monitor->instance()->passive);
+	return "ERROR: This command is only allowed in passive mode" unless (MMM::Monitor::Monitor->instance()->is_passive);
 
 	my $agents	= MMM::Monitor::Agents->instance();
 	my $roles	= MMM::Monitor::Roles->instance();
@@ -239,14 +239,19 @@
 	my $role	= shift;
 	my $host	= shift;
 	
-	return "ERROR: This command is only allowed in active mode" if (MMM::Monitor::Monitor->instance()->passive);
+	my $monitor	= MMM::Monitor::Monitor->instance();
+	return "ERROR: This command is not allowed in passive mode" if ($monitor->is_passive);
 
 	my $agents	= MMM::Monitor::Agents->instance();
 	my $roles	= MMM::Monitor::Roles->instance();
 
 	return "ERROR: Unknown role name '$role'!" unless ($roles->exists($role));
 	return "ERROR: Unknown host name '$host'!" unless ($agents->exists($host));
-	return "ERROR: move_role may be used for exclusive roles only!" unless ($roles->is_exclusive($role));
+
+	unless ($roles->is_exclusive($role)) {
+		$roles->clear_balanced_role($host, $role);
+		return "OK: Balanced role $role has been removed from host '$host'. Now you can wait some time and check new roles info!";
+	}
 
 	my $host_state = $agents->state($host);
 	return "ERROR: Can't move role to host with state $host_state." unless ($host_state eq 'ONLINE');
@@ -261,7 +266,9 @@
 	my $agent = MMM::Monitor::Agents->instance()->get($host);
 	return "ERROR: Can't reach agent daemon on '$host'! Can't move roles there!" unless ($agent->cmd_ping());
 
-	return "ERROR: Role '$role' is assigned to preferred host '$old_owner'. Can't move it!" if ($roles->assigned_to_preferred_host($role));
+	if ($monitor->is_active && $roles->assigned_to_preferred_host($role)) {
+		return "ERROR: Role '$role' is assigned to preferred host '$old_owner'. Can't move it!";
+	}
 
 	my $ip = $roles->get_exclusive_role_ip($role);
 	return "Error: Role $role has no IP." unless ($ip);
@@ -272,13 +279,13 @@
 	$roles->set_role($role, $ip, $host);
 
 	# Notify old host (if is_active_master_role($role) this will make the host non writable)
-	MMM::Monitor::Monitor->instance()->send_agent_status($old_owner);
+	$monitor->send_agent_status($old_owner);
 
 	# Notify slaves (this will make them switch the master)
-	MMM::Monitor::Monitor->instance()->notify_slaves($host) if ($roles->is_active_master_role($role));
+	$monitor->notify_slaves($host) if ($roles->is_active_master_role($role));
 
 	# Notify new host (if is_active_master_role($role) this will make the host writable)
-	MMM::Monitor::Monitor->instance()->send_agent_status($host);
+	$monitor->send_agent_status($host);
 	
 	return "OK: Role '$role' has been moved from '$old_owner' to '$host'. Now you can wait some time and check new roles info!";
 	
@@ -288,7 +295,8 @@
 	my $role	= shift;
 	my $host	= shift;
 	
-	return "ERROR: This command is only allowed in active mode" if (MMM::Monitor::Monitor->instance()->passive);
+	my $monitor	= MMM::Monitor::Monitor->instance();
+	return "ERROR: This command is not allowed in passive mode" if (MMM::Monitor::Monitor->instance()->is_passive);
 
 	my $agents	= MMM::Monitor::Agents->instance();
 	my $roles	= MMM::Monitor::Roles->instance();
@@ -328,12 +336,12 @@
 	if (!$checks->rep_threads($old_owner)) {
 		FATAL "State of host '$old_owner' changed from ONLINE to REPLICATION_FAIL (because of move_role --force)";
 		$old_agent->state('REPLICATION_FAIL');
-		$roles->clear_host_roles($old_owner);
+		$roles->clear_roles($old_owner) if ($monitor->is_active);
 	}
 	elsif (!$checks->rep_backlog($old_owner)) {
 		FATAL "State of host '$old_owner' changed from ONLINE to REPLICATION_BACKLOG (because of move_role --force)";
 		$old_agent->state('REPLICATION_BACKLOG');
-		$roles->clear_host_roles($old_owner);
+		$roles->clear_roles($old_owner) if ($monitor->is_active);
 	}
 
 	# Notify old host (this will make the host non writable)
@@ -352,13 +360,13 @@
 
 =item mode
 
-Get information about current mode (active or passive)
+Get information about current mode (active, manual or passive)
 
 =cut
 
 sub mode() {
-	return 'PASSIVE' if (MMM::Monitor::Monitor->instance()->passive);
-	return 'ACTIVE';
+	my $monitor	= MMM::Monitor::Monitor->instance();
+	return $monitor->get_mode_string();
 }
 
 
@@ -369,26 +377,69 @@
 =cut
 
 sub set_active() {
-	return 'OK: Already in active mode.' unless (MMM::Monitor::Monitor->instance()->passive);
-
-
-	# Send status to agents
-	MMM::Monitor::Monitor->instance()->send_status_to_agents();
-
-	# Clear 'bad' roles
-	my $agents	= MMM::Monitor::Agents->instance();
-	foreach my $host (keys(%{$main::config->{host}})) {
-		my $agent = $agents->get($host);
-		$agent->cmd_clear_bad_roles(); # TODO check result
-	}
-
-
-	MMM::Monitor::Monitor->instance()->passive(0);
-	MMM::Monitor::Monitor->instance()->passive_info('');
+	my $monitor	= MMM::Monitor::Monitor->instance();
+
+	return 'OK: Already in active mode.' if ($monitor->is_active);
+
+	my $old_mode = $monitor->get_mode_string();
+	INFO "Admin changed mode from '$old_mode' to 'ACTIVE'";
+	
+	if ($monitor->is_passive) {
+		$monitor->set_active(); # so that we can send status to agents
+		$monitor->cleanup_and_send_status();
+		$monitor->passive_info('');
+	}
+	elsif ($monitor->is_manual) {
+		# remove all roles from hosts which are not ONLINE
+		my $roles	= MMM::Monitor::Roles->instance();
+		my $agents	= MMM::Monitor::Agents->instance();
+		my $checks	= MMM::Monitor::ChecksStatus->instance();
+		foreach my $host (keys(%{$main::config->{host}})) {
+			my $host_state = $agents->state($host);
+			next if ($host_state eq 'ONLINE' || $roles->get_host_roles($host) == 0);
+			my $agent = $agents->get($host);
+			$roles->clear_roles($host);
+			my $ret = $monitor->send_agent_status($host);
+#			next if ($host_state eq 'REPLICATION_FAIL');
+#			next if ($host_state eq 'REPLICATION_BACKLOG');
+			# NOTE host_state should never be ADMIN_OFFLINE at this point
+			if (!$ret) {
+				ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
+				$monitor->_kill_host($host, $checks->ping($host));
+			}
+		}
+	}
+
+	$monitor->set_active();
 	return 'OK: Switched into active mode.';
 }
 
 
+=item set_manual
+
+Switch to manual mode.
+
+=cut
+
+sub set_manual() {
+	my $monitor	= MMM::Monitor::Monitor->instance();
+
+	return 'OK: Already in manual mode.' if ($monitor->is_manual);
+
+	my $old_mode = $monitor->get_mode_string();
+	INFO "Admin changed mode from '$old_mode' to 'MANUAL'";
+
+	if ($monitor->is_passive) {
+		$monitor->set_manual(); # so that we can send status to agents
+		$monitor->cleanup_and_send_status();
+		$monitor->passive_info('');
+	}
+
+	$monitor->set_manual();
+	return 'OK: Switched into manual mode.';
+}
+
+
 =item set_passive
 
 Switch to passive mode.
@@ -396,10 +447,15 @@
 =cut
 
 sub set_passive() {
-	return 'OK: Already in passive mode.' if (MMM::Monitor::Monitor->instance()->passive);
-
-	MMM::Monitor::Monitor->instance()->passive(1);
-	MMM::Monitor::Monitor->instance()->passive_info('Admin switched to passive mode.');
+	my $monitor	= MMM::Monitor::Monitor->instance();
+
+	return 'OK: Already in passive mode.' if ($monitor->is_passive);
+
+	my $old_mode = $monitor->get_mode_string();
+	INFO "Admin changed mode from '$old_mode' to 'PASSIVE'";
+
+	$monitor->set_passive();
+	$monitor->passive_info('Admin switched to passive mode.');
 	return 'OK: Switched into passive mode.';
 }
 
@@ -413,6 +469,7 @@
     set_offline <host>                - set host <host> offline
     mode                              - print current mode.
     set_active                        - switch into active mode.
+    set_manual                        - switch into manual mode.
     set_passive                       - switch into passive mode.
     move_role [--force] <role> <host> - move exclusive role <role> to host <host>
                                         (Only use --force if you know what you are doing!)

=== modified file 'lib/Monitor/Monitor.pm'
--- lib/Monitor/Monitor.pm	2010-02-11 01:05:09 +0000
+++ lib/Monitor/Monitor.pm	2010-03-09 10:25:26 +0000
@@ -19,6 +19,7 @@
 use MMM::Monitor::NetworkChecker;
 use MMM::Monitor::Role;
 use MMM::Monitor::Roles;
+use MMM::Monitor::StartupStatus;
 
 =head1 NAME
 
@@ -28,6 +29,11 @@
 
 our $VERSION = '0.01';
 
+use constant MMM_MONITOR_MODE_PASSIVE => 0;
+use constant MMM_MONITOR_MODE_ACTIVE  => 1;
+use constant MMM_MONITOR_MODE_MANUAL  => 2;
+use constant MMM_MONITOR_MODE_WAIT    => 3;
+
 use Class::Struct;
 
 sub instance() {
@@ -40,12 +46,13 @@
 	command_queue		=> 'Thread::Queue',
 	result_queue		=> 'Thread::Queue',
 	roles				=> 'MMM::Monitor::Roles',
-	passive				=> '$',
+	mode				=> '$',
 	passive_info		=> '$',
 	kill_host_bin		=> '$'
 };
 
 
+
 =head1 FUNCTIONS
 
 =over 4
@@ -59,6 +66,24 @@
 sub init($) {
 	my $self = shift;
 
+	#___________________________________________________________________________
+	#
+	# Wait until network connection is available
+	#___________________________________________________________________________
+
+	INFO "Waiting for network connection...";
+	unless (MMM::Monitor::NetworkChecker->wait_for_network()) {
+		INFO "Received shutdown request while waiting for network connection.";
+		return 0;
+	}
+	INFO "Network connection is available.";
+
+
+	#___________________________________________________________________________
+	#
+	# Create thread queues and other stuff... 
+	#___________________________________________________________________________
+
 	my $agents = MMM::Monitor::Agents->instance();
 
 	$self->checker_queue(new Thread::Queue::);
@@ -68,6 +93,23 @@
 	$self->roles(MMM::Monitor::Roles->instance());
 	$self->passive_info('');
 
+	if ($main::config->{monitor}->{mode} eq 'active') {
+		$self->mode(MMM_MONITOR_MODE_ACTIVE);
+	}
+	elsif ($main::config->{monitor}->{mode} eq 'manual') {
+		$self->mode(MMM_MONITOR_MODE_MANUAL);
+	}
+	elsif ($main::config->{monitor}->{mode} eq 'wait') {
+		$self->mode(MMM_MONITOR_MODE_WAIT);
+	}
+	elsif ($main::config->{monitor}->{mode} eq 'passive') {
+		$self->mode(MMM_MONITOR_MODE_PASSIVE);
+		$self->passive_info('Configured to start up in passive mode.');
+	}
+	else {
+		LOGDIE "Something very, very strange just happend - dieing..."
+	}
+
 
 	#___________________________________________________________________________
 	#
@@ -89,14 +131,6 @@
 
 	my $checks	= $self->checks_status;
 
-	#___________________________________________________________________________
-	#
-	# Go into passive mode if we have no network connection at startup
-	#___________________________________________________________________________
-
-	$self->passive(!$main::have_net);
-	$self->passive_info('No network connection during startup.') unless ($main::have_net);
-
 	
 	#___________________________________________________________________________
 	#
@@ -108,21 +142,21 @@
 
 	#___________________________________________________________________________
 	#
-	# Figure out current status. Go into passive mode if there are discrepancies
+	# Fetch stored status, agent status and system status
 	#___________________________________________________________________________
 
-	$agents->load_status();
-
-	my $system_status	= {};
-	my $agent_status	= {};
-	my $status			= 1;
+	$agents->load_status();	# load stored status
+
+
+	my $startup_status	= new MMM::Monitor::StartupStatus; 
+
 	my $res;
 
 	foreach my $host (keys(%{$main::config->{host}})) {
 
 		my $agent		= $agents->get($host);
-		my $host_status	= 1;
 
+		$startup_status->set_stored_status($host, $agent->state, $agent->roles);
 
 		#_______________________________________________________________________
 		#
@@ -132,28 +166,23 @@
 		$res = $agent->cmd_get_agent_status(2);
 
 		if ($res =~ /^OK/) {
-
 			my ($msg, $state, $roles_str, $master) = split('\|', $res);
 			my @roles_str_arr = sort(split(/\,/, $roles_str));
 			my @roles;
 
 			foreach my $role_str (@roles_str_arr) {
 				my $role = MMM::Monitor::Role->from_string($role_str);
-				if (defined($role)) {
-					push @roles, $role;
-				}
+				push(@roles, $role) if (defined($role));
 			}
 
-			$agent_status->{$host} = { state => $state, roles => \@roles, master => $master };
+			$startup_status->set_agent_status($host, $state, \@roles, $master);
 		}
 		elsif ($agent->state ne 'ADMIN_OFFLINE') {
 			if ($checks->ping($host) && $checks->mysql($host) && !$agent->agent_down()) {
 				ERROR "Can't reach agent on host '$host'";
 				$agent->agent_down(1);
 			}
-			ERROR "Switching to passive mode: The status of the agent on host '$host' could not be determined (answer was: $res).";
-			$status			= 0;
-			$host_status	= 0;
+			ERROR "The status of the agent on host '$host' could not be determined (answer was: $res).";
 		}
 		
 
@@ -163,180 +192,61 @@
 		#_______________________________________________________________________
 
 		$res = $agent->cmd_get_system_status(2);
+
 		if ($res =~ /^OK/) {
-			my ($msg, $writable, $roles_str) = split('\|', $res);
+			my ($msg, $writable, $roles_str, $master_ip) = split('\|', $res);
 			my @roles_str_arr = sort(split(/\,/, $roles_str));
 			my @roles;
+
 			foreach my $role_str (@roles_str_arr) {
 				my $role = MMM::Monitor::Role->from_string($role_str);
-				if (defined($role)) {
-					push @roles, $role;
+				push(@roles, $role) if (defined($role));
+			}
+
+			my $master = '';
+			if (defined($master_ip)) {
+			    foreach my $a_host (keys(%{$main::config->{host}})) {
+					$master = $a_host if ($main::config->{host}->{$a_host}->{ip} eq $master_ip);
 				}
 			}
-			$system_status->{$host} = {
-				writable	=> $writable,
-				roles		=> \@roles
-			};
+			$startup_status->set_system_status($host, $writable, \@roles, $master);
 		}
 		elsif ($agent->state ne 'ADMIN_OFFLINE') {
 			if ($checks->ping($host) && $checks->mysql($host) && !$agent->agent_down()) {
 				ERROR "Can't reach agent on host '$host'";
 				$agent->agent_down(1);
 			}
-			ERROR "Switching to passive mode: The status of the system '$host' could not be determined (answer was: $res).";
-			$status			= 0;
-			$host_status	= 0;
-
-		}
-
-
-		#_______________________________________________________________________
-		#
-		# Skip comparison, if we coult not fetch AGENT/SYSTEM status
-		#_______________________________________________________________________
-		
-		next unless (defined($agent_status->{$host}));
-		next unless (defined($system_status->{$host}));
-
-
-		#_______________________________________________________________________
-		#
-		# Compare agent and system status ...
-		#_______________________________________________________________________
-
-		if ($agent_status->{$host}->{state} ne 'UNKNOWN' && $agent_status->{$host}->{state} ne $agent->state) {
-			ERROR "Switching to passive mode: Agent state '", $agent_status->{$host}->{state}, "' differs from stored one '", $agent->state, "' for host '$host'.";
-			$status			= 0;
-			$host_status	= 0;
-			next;
-		}
-
-
-		#_______________________________________________________________________
-		#
-		# ... determine if roles differ 
-		#_______________________________________________________________________
-
-		my $changes	= 0;
-		my $diff	= new Algorithm::Diff:: (
-			$system_status->{$host}->{roles},
-			$agent->roles,
-			{ keyGen => \&MMM::Common::Role::to_string }
-		);
-
-		while ($diff->Next) {
-			next if ($diff->Same);
-
-			ERROR sprintf(
-				"Switching to passive mode: Roles of host '$host' [%s] differ from stored ones [%s]",
-				join(', ', @{$system_status->{$host}->{roles}}),
-				join(', ', @{$agent->roles})
-			);
-			$status			= 0;
-			$host_status	= 0;
-			last;
-		}
-		
-		next unless ($host_status);
-		foreach my $role (@{$agent->roles}) {
-			next unless ($self->roles->is_active_master_role($role->name));
-			next if ($system_status->{$host}->{writable});
-			WARN "Active master $host was not writable at monitor startup. (Don't mind, the host will be made writable soon)"
-		}
-		
-	}
-
-	DEBUG "STATE INFO\n", Data::Dumper->Dump([$agents, $agent_status, $system_status], ['Stored status', 'Agent status', 'System status']);
-
-
-	#___________________________________________________________________________
-	#
-	# Maybe switch into passive mode?
-	#___________________________________________________________________________
-
-	unless ($status) {
-		# Enter PASSIVE MODE
-		$self->passive(1);
-		my $agent_status_str = '';
-		foreach my $host (sort(keys(%{$agent_status}))) {
-			$agent_status_str .= sprintf(
-				"  %s %s. Roles: %s. Master: %s\n",
-				$host,
-				$agent_status->{$host}->{state},
-				scalar(@{$agent_status->{$host}->{roles}}) > 0 ? join(', ', sort(@{$agent_status->{$host}->{roles}})) : 'none',
-				$agent_status->{$host}->{master} ? $agent_status->{$host}->{master} : '?'
-			);
-		}
-		my $system_status_str = '';
-		foreach my $host (sort(keys(%{$system_status}))) {
-			$system_status_str .= sprintf(
-				"  %s %s. Roles: %s\n",
-				$host,
-				$system_status->{$host}->{writable} ? 'writable' : 'readonly',
-				scalar(@{$system_status->{$host}->{roles}}) > 0 ? join(', ', sort(@{$system_status->{$host}->{roles}})) : 'none'
-			);
-		}
-		my $status_str = sprintf("\nStored status:\n%s\nAgent status:\n%s\nSystem status:\n%s", $agents->get_status_info(), $agent_status_str, $system_status_str);
-		$self->passive_info("Discrepancies between stored status, agent status and system status during startup.\n" . $status_str);
-		FATAL "Switching to passive mode now. See output of 'mmm_control show' for details.";
-		INFO $status_str;
-
-		foreach my $host (keys(%{$main::config->{host}})) {
-			my $agent = $agents->get($host);
-
-			# Set all unknown hosts to AWAITING_RECOVERY
-			$agent->state('AWAITING_RECOVERY') if ($agent->state eq 'UNKNOWN');
-
-			next unless ($system_status->{$host});
-			next unless (scalar(@{$system_status->{$host}->{roles}}));
-			# Set status restored from agent systems
-			$agent->state('ONLINE');
-			foreach my $role (@{$system_status->{$host}->{roles}}) {
-				next unless ($self->roles->exists_ip($role->name, $role->ip));
-				next unless ($self->roles->can_handle($role->name, $host));
-				$self->roles->set_role($role->name, $role->ip, $host);
-			}
-		}
-
-		# propagate roles to agent objects
-		foreach my $host (keys(%{$main::config->{host}})) {
-			my $agent = $agents->get($host);
-			my @roles = sort($self->roles->get_host_roles($host));
-			$agent->roles(\@roles);
-		}
-
-		WARN "Monitor started in passive mode.";
-
-		return;
-	}
-
-	# Stay in ACTIVE MODE
-	# Everything is okay, apply roles from status file.
-	foreach my $host (keys(%{$main::config->{host}})) {
+			ERROR "The status of the system '$host' could not be determined (answer was: $res).";
+		}
+	}
+
+	my $conflict = $startup_status->determine_status();
+
+	DEBUG "STATE INFO\n", Data::Dumper->Dump([$startup_status], ['Startup status']);
+	INFO $startup_status->to_string();
+
+	foreach my $host (keys(%{$startup_status->{result}})) {
 		my $agent = $agents->get($host);
-
-		# Set new hosts to AWAITING_RECOVERY
-		if ($agent->state eq 'UNKNOWN') {
-			WARN "Detected new host '$host': Setting its initial state to 'AWAITING_RECOVERY'. Use 'mmm_control set_online $host' to switch it online.";
-			$agent->state('AWAITING_RECOVERY');
-		}
-
-		# Apply roles loaded from status file
-		foreach my $role (@{$agent->roles}) {
-			unless ($self->roles->exists_ip($role->name, $role->ip)) {
-				WARN "Detected change in role definitions: Role '$role' was removed.";
-				next;
-			}
-			unless ($self->roles->can_handle($role->name, $host)) {
-				WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
-				next;
-			}
+		$agent->state($startup_status->{result}->{$host}->{state});
+		foreach my $role (@{$startup_status->{result}->{$host}->{roles}}) {
 			$self->roles->set_role($role->name, $role->ip, $host);
 		}
 	}
 
-	INFO "Monitor started in active mode."  unless ($self->passive);
-	WARN "Monitor started in passive mode." if ($self->passive);
+	if ($conflict && $main::config->{monitor}->{careful_startup}) {
+		$self->set_passive();
+		$self->passive_info("Conflicting roles during startup:\n\n" . $startup_status->to_string());
+	}
+	elsif (!$self->is_passive) {
+		$self->cleanup_and_send_status();
+	}
+	
+	INFO "Monitor started in active mode."  if ($self->mode == MMM_MONITOR_MODE_ACTIVE);
+	INFO "Monitor started in manual mode."  if ($self->mode == MMM_MONITOR_MODE_MANUAL);
+	INFO "Monitor started in wait mode."    if ($self->mode == MMM_MONITOR_MODE_WAIT);
+	INFO "Monitor started in passive mode." if ($self->mode == MMM_MONITOR_MODE_PASSIVE);
+
+	return 1;
 }
 
 sub check_master_configuration($) {
@@ -507,7 +417,7 @@
 
 	foreach my $host (keys(%{$main::config->{host}})) {
 
-		$agents->save_status() unless ($self->passive);
+		$agents->save_status() unless ($self->is_passive);
 
 		my $agent		= $agents->get($host);
 		my $state		= $agent->state;
@@ -539,7 +449,8 @@
 			unless ($ping && $mysql) {
 				FATAL sprintf("State of host '%s' changed from %s to HARD_OFFLINE (ping: %s, mysql: %s)", $host, $state, ($ping? 'OK' : 'not OK'), ($mysql? 'OK' : 'not OK'));
 				$agent->state('HARD_OFFLINE');
-				$self->roles->clear_host_roles($host);
+				next if ($self->is_manual);
+				$self->roles->clear_roles($host);
 				if (!$self->send_agent_status($host)) {
 					ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
 					$self->_kill_host($host, $checks->ping($host));
@@ -557,8 +468,12 @@
 			if ($ping && $mysql && !$rep_threads && $peer_state eq 'ONLINE' && $checks->ping($peer) && $checks->mysql($peer)) {
 				FATAL "State of host '$host' changed from $state to REPLICATION_FAIL";
 				$agent->state('REPLICATION_FAIL');
-				$self->roles->clear_host_roles($host);
-				$self->send_agent_status($host);
+				next if ($self->is_manual);
+				$self->roles->clear_roles($host);
+				if (!$self->send_agent_status($host)) {
+					ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
+					$self->_kill_host($host, $checks->ping($host));
+				}
 				next;
 			}
 
@@ -566,8 +481,12 @@
 			if ($ping && $mysql && !$rep_backlog && $rep_threads && $peer_state eq 'ONLINE' && $checks->ping($peer) && $checks->mysql($peer)) {
 				FATAL "State of host '$host' changed from $state to REPLICATION_DELAY";
 				$agent->state('REPLICATION_DELAY');
-				$self->roles->clear_host_roles($host);
-				$self->send_agent_status($host);
+				next if ($self->is_manual);
+				$self->roles->clear_roles($host);
+				if (!$self->send_agent_status($host)) {
+					ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host);
+					$self->_kill_host($host, $checks->ping($host));
+				}
 				next;
 			}
 			next;
@@ -711,7 +630,47 @@
 			next;
 		}
 	}
-	$agents->save_status() unless ($self->passive);
+
+	if ($self->mode == MMM_MONITOR_MODE_WAIT) {
+		my $master_one	= $self->roles->get_first_master();
+		my $master_two	= $self->roles->get_second_master();
+		my $state_one	= $agents->state($master_one);
+		my $state_two	= $agents->state($master_two);
+
+		if ($state_one eq 'ONLINE' && $state_two eq 'ONLINE') {
+			INFO "Nodes $master_one and $master_two are ONLINE, switching from mode 'WAIT' to 'ACTIVE'.";
+			$self->set_active();
+		}
+		elsif ($main::config->{monitor}->{wait_for_other_master} > 0 && ($state_one eq 'ONLINE' || $state_two eq 'ONLINE')) {
+			my $living_master = $state_one eq 'ONLINE' ? $master_one : $master_two;
+			my $dead_master   = $state_one eq 'ONLINE' ? $master_two : $master_one;
+
+			if ($main::config->{monitor}->{wait_for_other_master} <= time() - $agents->online_since($living_master)) {
+				$self->set_active();
+				WARN sprintf("Master $dead_master did not come online for %d(wait_for_other_master) seconds. Switching from mode 'WAIT' to 'ACTIVE'", $main::config->{monitor}->{wait_for_other_master});
+			}
+
+		}
+		if ($self->is_active) {
+			# cleanup
+			foreach my $host (keys(%{$main::config->{host}})) {
+				my $host_state = $agents->state($host);
+				next if ($host_state eq 'ONLINE' || $self->roles->get_host_roles($host) == 0); 
+				my $agent = $agents->get($host); 
+				$self->roles->clear_roles($host); 
+				my $ret = $self->send_agent_status($host); 
+#   			next if ($host_state eq 'REPLICATION_FAIL'); 
+#   			next if ($host_state eq 'REPLICATION_BACKLOG'); 
+				# NOTE host_state should never be ADMIN_OFFLINE at this point 
+				if (!$ret) { 
+					ERROR sprintf("Can't send offline status notification to '%s' - killing it!", $host); 
+					$self->_kill_host($host, $checks->ping($host)); 
+				} 
+			}
+		}
+	}
+
+	$agents->save_status() unless ($self->is_passive);
 }
 
 
@@ -725,7 +684,7 @@
 	my $self = shift;
 
 	# Never change roles if we are in PASSIVE mode
-	return if ($self->passive);
+	return if ($self->is_passive);
 
 	my $old_active_master = $self->roles->get_active_master();
 	
@@ -734,7 +693,7 @@
 	$self->roles->process_orphans('balanced');
 
 	# obey preferences
-	$self->roles->obey_preferences();
+	$self->roles->obey_preferences() if ($self->is_active);
 
 	# Balance roles
 	$self->roles->balance();
@@ -749,6 +708,46 @@
 }
 
 
+=item cleanup_and_send_status()
+
+Send status information to all agents and clean up old roles.
+
+=cut
+sub cleanup_and_send_status($) {
+	my $self	= shift;
+
+	my $agents = MMM::Monitor::Agents->instance();
+	my $roles = MMM::Monitor::Roles->instance();
+
+	my $active_master  = $roles->get_active_master();
+	my $passive_master = $roles->get_passive_master();
+
+	# Notify passive master first
+	if ($passive_master ne '') {
+		my $host = $passive_master;
+		$self->send_agent_status($host);
+		my $agent = $agents->get($host);
+		$agent->cmd_clear_bad_roles(); # TODO check result
+	}
+
+	# Notify all slave hosts
+	foreach my $host (keys(%{$main::config->{host}})) {
+		next if ($self->roles->is_master($host));
+		$self->send_agent_status($host);
+		my $agent = $agents->get($host);
+		$agent->cmd_clear_bad_roles(); # TODO check result
+	}
+
+	# Notify active master at the end
+	if ($active_master ne '') {
+		my $host = $active_master;
+		$self->send_agent_status($host);
+		my $agent = $agents->get($host);
+		$agent->cmd_clear_bad_roles(); # TODO check result
+	}
+}
+
+
 =item send_status_to_agents
 
 Send status information to all agents.
@@ -797,7 +796,7 @@
 
 	# Never send anything to agents if we are in PASSIVE mode
 	# Never send anything to agents if we have no network connection
-	return if ($self->passive || !$main::have_net);
+	return if ($self->is_passive || !$main::have_net);
 
 	# Determine active master if it was not passed
 	$master = $self->roles->get_active_master() unless (defined($master));
@@ -903,6 +902,7 @@
 		elsif ($command eq 'mode'			&& $arg_cnt == 0) { $res = MMM::Monitor::Commands::mode();							}
 		elsif ($command eq 'set_active'		&& $arg_cnt == 0) { $res = MMM::Monitor::Commands::set_active();					}
 		elsif ($command eq 'set_passive'	&& $arg_cnt == 0) { $res = MMM::Monitor::Commands::set_passive();					}
+		elsif ($command eq 'set_manual'		&& $arg_cnt == 0) { $res = MMM::Monitor::Commands::set_manual();					}
 		elsif ($command eq 'set_online'		&& $arg_cnt == 1) { $res = MMM::Monitor::Commands::set_online ($args[0]);			}
 		elsif ($command eq 'set_offline'	&& $arg_cnt == 1) { $res = MMM::Monitor::Commands::set_offline($args[0]);			}
 		elsif ($command eq 'move_role'		&& $arg_cnt == 2) { $res = MMM::Monitor::Commands::move_role($args[0], $args[1]);	}
@@ -917,5 +917,93 @@
 	}
 }
 
+
+=item is_active()
+
+Check if monitor is in active mode
+
+=cut
+
+sub is_active($$) {
+	my $self	= shift;
+	return ($self->mode == MMM_MONITOR_MODE_ACTIVE);
+}
+
+
+=item is_manual()
+
+Check if monitor is in manual mode
+
+=cut
+
+sub is_manual($$) {
+	my $self	= shift;
+	return ($self->mode == MMM_MONITOR_MODE_MANUAL || $self->mode == MMM_MONITOR_MODE_WAIT);
+}
+
+
+=item is_passive()
+
+Check if monitor is in passive mode
+
+=cut
+
+sub is_passive($$) {
+	my $self	= shift;
+	return ($self->mode == MMM_MONITOR_MODE_PASSIVE);
+}
+
+
+=item set_active()
+
+Set mode to active
+
+=cut
+
+sub set_active($$) {
+	my $self	= shift;
+	$self->mode(MMM_MONITOR_MODE_ACTIVE);
+}
+
+
+=item set_manual()
+
+Set mode to manual
+
+=cut
+
+sub set_manual($$) {
+	my $self	= shift;
+	$self->mode(MMM_MONITOR_MODE_MANUAL);
+}
+
+
+=item set_passive()
+
+Set mode to passive
+
+=cut
+
+sub set_passive($$) {
+	my $self	= shift;
+	$self->mode(MMM_MONITOR_MODE_PASSIVE);
+}
+
+
+=item get_mode_string()
+
+Get string representation of current mode
+
+=cut
+
+sub get_mode_string($) {
+	my $self	= shift;
+	return 'ACTIVE'  if ($self->mode == MMM_MONITOR_MODE_ACTIVE);
+	return 'MANUAL'  if ($self->mode == MMM_MONITOR_MODE_MANUAL);
+	return 'WAIT'    if ($self->mode == MMM_MONITOR_MODE_WAIT);
+	return 'PASSIVE' if ($self->mode == MMM_MONITOR_MODE_PASSIVE);
+	return 'UNKNOWN'; # should never happen
+}
+
 1;
 

=== modified file 'lib/Monitor/NetworkChecker.pm'
--- lib/Monitor/NetworkChecker.pm	2009-02-10 08:18:57 +0000
+++ lib/Monitor/NetworkChecker.pm	2010-03-09 10:25:26 +0000
@@ -54,29 +54,32 @@
 	$checker->shutdown();
 }
 
-sub initial_check() {
+sub wait_for_network() {
 	my @ips		= @{$main::config->{monitor}->{ping_ips}};
-	my $state	= 0;
 	
 	# Create checker
 	my $checker = new MMM::Monitor::Checker::('ping_ip');
 
-	# Ping all ips
-	foreach my $ip (@ips) {
-		# Ping checker
-		$checker->spawn() unless $checker->ping();
-
-		my $res = $checker->check($ip);
-		if ($res =~ /^OK/) {
-			DEBUG "IP '$ip' is reachable: $res";
-			$state = 1;
-			last;
+	while (!$main::shutdown) {
+		# Ping all ips
+		foreach my $ip (@ips) {
+			last if ($main::shutdown);
+			# Ping checker
+			$checker->spawn() unless $checker->ping();
+	
+			my $res = $checker->check($ip);
+			if ($res =~ /^OK/) {
+				DEBUG "IP '$ip' is reachable: $res";
+				$checker->shutdown();
+				return 1;
+			}
 		}
-		DEBUG "IP '$ip' is not reachable: $res";
+
+		# Sleep a while before checking every ip again
+		sleep($main::config->{monitor}->{ping_interval});
 	}
 	$checker->shutdown();
-
-	return $state;
+	return 0;
 }
 
 1;

=== modified file 'lib/Monitor/Roles.pm'
--- lib/Monitor/Roles.pm	2009-10-29 15:27:32 +0000
+++ lib/Monitor/Roles.pm	2010-03-09 10:25:26 +0000
@@ -112,6 +112,29 @@
 }
 
 
+=item host_has_roles($host)
+
+Check whether there are roles assigned to host $host
+
+=cut
+
+sub host_has_roles($$) {
+	my $self	= shift;
+	my $host	= shift;
+
+	return 0 unless (defined($host));
+
+	foreach my $role (keys(%$self)) {
+		my $role_info = $self->{$role};
+		foreach my $ip (keys(%{$role_info->{ips}})) {
+			my $ip_info = $role_info->{ips}->{$ip};
+			return 1 if ($ip_info->{assigned_to} eq $host);
+		}
+	}
+	return 0;
+}
+
+
 =item count_host_roles($host)
 
 Count all roles assigned to host $host
@@ -155,6 +178,74 @@
 }
 
 
+=item get_passive_master
+
+Get the passive master
+
+=cut
+
+sub get_passive_master($) {
+	my $self	= shift;
+
+	my $role = $self->{$main::config->{active_master_role}};
+	my $active_master = $self->get_active_master();
+	return '' unless $role;
+	return '' unless $active_master;
+
+	foreach my $host ( @{ $role->{hosts} } ) {
+		return $host if ($host ne $active_master);
+	}
+	return '';
+}
+
+
+=item get_first_master
+
+Get the first master
+
+=cut
+
+sub get_first_master($) {
+	my $self	= shift;
+
+	my $role = $self->{$main::config->{active_master_role}};
+	return '' unless $role;
+	return '' unless $role->{hosts}[0];
+	return $role->{hosts}[0];
+}
+
+
+=item get_second_master
+
+Get the second master
+
+=cut
+
+sub get_second_master($) {
+	my $self	= shift;
+
+	my $role = $self->{$main::config->{active_master_role}};
+	return '' unless $role;
+	return '' unless $role->{hosts}[1];
+	return $role->{hosts}[1];
+}
+
+
+=item get_master_hosts
+
+Get the hosts which can handle the active master-role
+
+=cut
+
+sub get_master_hosts($) {
+	my $self	= shift;
+
+	my $role = $self->{$main::config->{active_master_role}};
+	return '' unless $role;
+	return $self->{$role}->{hosts};
+}
+
+
 =item get_exclusive_role_owner($role)
 
 Get the host which has the exclusive role $role assigned
@@ -211,13 +302,13 @@
 }
 
 
-=item clear_host_roles($host)
+=item clear_roles($host)
 
 Remove all roles from host $host.
 
 =cut
 
-sub clear_host_roles($$) {
+sub clear_roles($$) {
 	my $self	= shift;
 	my $host	= shift;
 
@@ -238,6 +329,34 @@
 }
 
 
+=item clear_balanced_role($host, $role)
+
+Remove balanced role $role from host $host.
+
+=cut
+
+sub clear_balanced_role($$$) {
+	my $self	= shift;
+	my $host	= shift;
+	my $role	= shift;
+
+	INFO "Removing balanced role $role from host '$host':";
+
+	my $role_info = $self->{$role};
+	return 0 unless $role_info;
+	my $cnt = 0;
+	next unless ($role_info->{mode} eq 'balanced');
+	foreach my $ip (keys(%{$role_info->{ips}})) {
+		my $ip_info = $role_info->{ips}->{$ip};
+		next unless ($ip_info->{assigned_to} eq $host);
+		$cnt++;
+		INFO "    Removed role '$role($ip)' from host '$host'";
+		$ip_info->{assigned_to} = '';
+	}
+	return $cnt;
+}
+
+
 =item find_eligible_host($role)
 
 find host which can take over the role $role
@@ -562,6 +681,21 @@
 }
 
 
+=item is_master($host)
+
+Check if host $host can handle role $role.
+
+=cut
+
+sub is_master($$) {
+	my $self	= shift;
+	my $host	= shift;
+	my $role = $self->{$main::config->{active_master_role}};
+	return 0 unless defined($role);
+	return grep({$_ eq $host} @{$role->{hosts}});
+}
+
+
 =item is_active_master_role($role)
 
 Check whether $role is the active master role.

=== added file 'lib/Monitor/StartupStatus.pm'
--- lib/Monitor/StartupStatus.pm	1970-01-01 00:00:00 +0000
+++ lib/Monitor/StartupStatus.pm	2010-03-09 10:25:26 +0000
@@ -0,0 +1,298 @@
+package MMM::Monitor::StartupStatus;
+
+use strict;
+use warnings FATAL => 'all';
+use List::Util qw(max);
+use Log::Log4perl qw(:easy);
+use MMM::Common::Role;
+use MMM::Monitor::Role;
+use MMM::Monitor::Roles;
+
+our $VERSION = '0.01';
+
+=head1 NAME
+
+MMM::Monitor::StartupStatus - holds information about agent/system/stored status during startup
+
+=cut
+
+sub new($) {
+	my $class   = shift;
+
+	my $self = {
+		roles => {},
+		hosts => {},
+		result=> {}
+	};
+	return bless $self, $class;
+}
+
+
+=head1 FUNCTIONS
+
+=over 4
+
+=item set_agent_status($host, $state, $roles, $master)
+
+Set agent status
+
+=cut
+
+sub set_agent_status($$\@$) {
+	my $self	= shift;
+	my $host	= shift;
+	my $state	= shift;
+	my $roles	= shift;
+	my $master	= shift;
+
+	$self->{hosts}->{$host}				= {} unless (defined($self->{hosts}->{$host}));
+	$self->{hosts}->{$host}->{agent}	= {
+		state	=> $state,
+		master	=> $master
+	};
+	foreach my $role (@{$roles}) {
+		unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) {
+			WARN "Detected change in role definitions: Role '$role' was removed.";
+			next;
+		}
+		unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) {
+			WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
+			next;
+		}
+		my $role_str = $role->to_string();
+		$self->{roles}->{$role_str}						= {} unless (defined($self->{roles}->{$role_str}));
+		$self->{roles}->{$role_str}->{$host}			= {} unless (defined($self->{roles}->{$role_str}->{$host}));
+		$self->{roles}->{$role_str}->{$host}->{agent}	= 1;
+	}
+}
+
+
+=item set_stored_status($host, $state, $roles)
+
+Set stored status
+
+=cut
+
+sub set_stored_status($$\@$) {
+	my $self	= shift;
+	my $host	= shift;
+	my $state	= shift;
+	my $roles	= shift;
+
+	$self->{hosts}->{$host}				= {} unless (defined($self->{hosts}->{$host}));
+	$self->{hosts}->{$host}->{stored}	= {
+		state	=> $state,
+	};
+	foreach my $role (@{$roles}) {
+		unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) {
+			WARN "Detected change in role definitions: Role '$role' was removed.";
+			next;
+		}
+		unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) {
+			WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
+			next;
+		}
+		my $role_str = $role->to_string();
+		$self->{roles}->{$role_str}						= {} unless (defined($self->{roles}->{$role_str}));
+		$self->{roles}->{$role_str}->{$host}			= {} unless (defined($self->{roles}->{$role_str}->{$host}));
+		$self->{roles}->{$role_str}->{$host}->{stored}	= 1;
+	}
+}
+
+
+=item set_system_status($host, $writable, $roles, $master)
+
+Set system status
+
+=cut
+
+sub set_system_status($$\@$) {
+	my $self	= shift;
+	my $host	= shift;
+	my $writable= shift;
+	my $roles	= shift;
+	my $master	= shift;
+
+	$self->{hosts}->{$host}				= {} unless (defined($self->{hosts}->{$host}));
+	$self->{hosts}->{$host}->{system}	= {
+		writable=> $writable,
+		master	=> $master
+	};
+	foreach my $role (@{$roles}) {
+		unless (MMM::Monitor::Roles->instance()->exists_ip($role->name, $role->ip)) {
+			WARN "Detected change in role definitions: Role '$role' was removed.";
+			next;
+		}
+		unless (MMM::Monitor::Roles->instance()->can_handle($role->name, $host)) {
+			WARN "Detected change in role definitions: Host '$host' can't handle role '$role' anymore.";
+			next;
+		}
+		my $role_str = $role->to_string();
+		$self->{roles}->{$role_str}						= {} unless (defined($self->{roles}->{$role_str}));
+		$self->{roles}->{$role_str}->{$host}			= {} unless (defined($self->{roles}->{$role_str}->{$host}));
+		$self->{roles}->{$role_str}->{$host}->{system}	= 1;
+	}
+}
+
+sub determine_status() {
+	my $self	= shift;
+	my $roles	= MMM::Monitor::Roles->instance();
+
+	my $is_manual = MMM::Monitor::Monitor->instance()->is_manual();
+
+	my $conflict = 0;
+
+    foreach my $host (keys(%{$main::config->{host}})) {
+
+		# Figure out host state
+
+		my $stored_state = 'UNKNOWN';
+		my $agent_state  = 'UNKNOWN';
+		my $state;
+
+		$stored_state = $self->{hosts}->{$host}->{stored}->{state} if (defined($self->{hosts}->{$host}->{stored}->{state}));
+		$agent_state  = $self->{hosts}->{$host}->{agent}->{state}  if (defined($self->{hosts}->{$host}->{agent}->{state} ));
+
+		if (   $stored_state eq 'ADMIN_OFFLINE'     || $agent_state eq 'ADMIN_OFFLINE'    ) { $state = 'ADMIN_OFFLINE';     }
+		elsif ($stored_state eq 'HARD_OFFLINE'      || $agent_state eq 'HARD_OFFLINE'     ) { $state = 'HARD_OFFLINE';      }
+		elsif ($stored_state eq 'REPLICATION_FAIL'  || $agent_state eq 'REPLICATION_FAIL' ) { $state = 'REPLICATION_FAIL';  }
+		elsif ($stored_state eq 'REPLICATION_DELAY' || $agent_state eq 'REPLICATION_DELAY') { $state = 'REPLICATION_DELAY'; }
+		elsif ($stored_state eq 'ONLINE'            || $agent_state eq 'ONLINE'           ) { $state = 'ONLINE';            }
+		else                                                                                { $state = 'AWAITING_RECOVERY'; }
+
+		$self->{result}->{$host} = { state => $state, roles => [] };
+	}
+
+    foreach my $role_str (keys(%{$self->{roles}})) {
+		my $role = MMM::Monitor::Role->from_string($role_str);
+		next unless(defined($role));
+
+		if ($roles->is_active_master_role($role->name)) {
+			# active master role
+			my $max        = 0;
+			my $target     = undef;
+			my $system_cnt = 0;
+			foreach my $host (keys(%{$self->{roles}->{$role_str}})) {
+				my $votes = 0;
+				my $info  = $self->{roles}->{$role_str}->{$host};
+				my $host_info = $self->{hosts}->{$host};
+
+				# host is writable
+				$votes += 4 if (defined($host_info->{system}->{writable}) && $host_info->{system}->{writable});
+
+				# IP is configured
+				if (defined($info->{system})) {
+					$votes += 2;
+					$system_cnt++;
+				}
+
+				$votes += 1 if (defined($info->{stored}));
+				$votes += 1 if (defined($info->{agent}));
+
+				foreach my $slave_host (keys(%{$self->{hosts}})) {
+					my $slave_info = $self->{hosts}->{$slave_host};
+					next if MMM::Monitor::Roles->instance()->is_master($slave_host);
+					$votes++ if (defined($slave_info->{system}->{master}) && $slave_info->{system}->{master} eq $host);
+				}
+	
+	
+				my $state = $self->{result}->{$host}->{state};
+				$votes = 0 if ($state eq 'ADMIN_OFFLINE');
+				$votes = 0 if ($state eq 'HARD_OFFLINE' && !$is_manual);
+
+				if ($votes > $max) {
+					$target = $host;
+					$max = $votes;
+				}
+			}
+			if ($system_cnt > 1) {
+				WARN "Role '$role_str' was configured on $system_cnt hosts during monitor startup.";
+				$conflict = 1;
+			}
+			if (defined($target)) { 
+				push (@{$self->{result}->{$target}->{roles}}, $role);
+				my $state = $self->{result}->{$target}->{state};
+				$self->{result}->{$target}->{state} = 'ONLINE' if (!$is_manual || $state eq 'REPLICATION_FAIL' || $state eq 'REPLICATION_DELAY');
+			}
+			next;
+		}
+
+		# Handle non-writer roles
+		my $max        = 0;
+		my $target     = undef;
+		my $system_cnt = 0;
+		foreach my $host (keys(%{$self->{roles}->{$role_str}})) {
+			my $votes = 0;
+			my $info  = $self->{roles}->{$role_str}->{$host};
+
+			# IP is configured
+			if (defined($info->{system})) {
+				$votes += 4;
+				$system_cnt++;
+			}
+
+			$votes += 2 if (defined($info->{stored}));
+			$votes += 1 if (defined($info->{agent}));
+
+
+			my $state = $self->{result}->{$host}->{state};
+			if ($state eq 'ADMIN_OFFLINE' || (!$is_manual && $state ne 'ONLINE' && $state ne 'AWAITING_RECOVERY')) {
+				$votes = 0;
+			}
+			if ($votes > $max) {
+				$target    = $host;
+				$max       = $votes;
+			}
+		}
+		if ($system_cnt > 1) {
+			WARN "Role '$role_str' was configured on $system_cnt hosts during monitor startup.";
+		}
+		if (defined($target)) { 
+			push (@{$self->{result}->{$target}->{roles}}, $role);
+			$self->{result}->{$target}->{state} = 'ONLINE' if ($self->{result}->{$target}->{state} eq 'AWAITING_RECOVERY');
+		}
+	}
+	return $conflict;
+}
+
+
+sub to_string($) {
+	my $self	= shift;
+	my $ret = "Startup status:\n";
+	$ret .= "\nRoles:\n";
+
+    my $role_len = 4; # "Role"
+    my $host_len = 6; # "Master"
+
+    foreach my $role (keys(%{$main::config->{role}})) { $role_len = max($role_len, length $role) }
+    foreach my $host (keys(%{$main::config->{host}})) { $host_len = max($host_len, length $host) }
+	$role_len += 17; # "(999.999.999.999)"
+
+	$ret .= sprintf("    %-*s  %-*s  %-6s  %-6s  %-5s\n", $role_len, 'Role', $host_len, 'Host', 'Stored', 'System', 'Agent');
+	foreach my $role (keys(%{$self->{roles}})) {
+		foreach my $host (keys(%{$self->{roles}->{$role}})) {
+			my $info = $self->{roles}->{$role}->{$host};
+			$ret .= sprintf("    %-*s  %-*s  %-6s  %-6s  %-5s\n", $role_len, $role, $host_len, $host,
+				defined($info->{stored}) ? 'Yes'  : '-',
+				defined($info->{system}) ? 'Yes'  : '-',
+				defined($info->{agent})  ? 'Yes'  : '-'
+			);
+		}
+	}
+
+	$ret .= "\nHosts:\n";
+	$ret .= sprintf("    %-*s  %-*s  %-8s  %-16s  %-16s\n", $host_len, 'Host', $host_len, 'Master', 'Writable', 'Stored state', 'Agent state');
+	foreach my $host (keys(%{$self->{hosts}})) {
+		my $info = $self->{hosts}->{$host};
+		my $is_master = MMM::Monitor::Roles->instance()->is_master($host);
+		$ret .= sprintf("    %-*s  %-*s  %-8s  %-16s  %-16s\n", $host_len, $host, $host_len,
+			$is_master ? '-' : (defined($info->{system}->{master})   ? $info->{system}->{master}   : '?'),
+			defined($info->{system}->{writable}) ? ($info->{system}->{writable} ? 'Yes' : 'No') : '?',
+			defined($info->{stored}->{state})    ? $info->{stored}->{state} : '?',
+			defined($info->{agent}->{state})     ? $info->{agent}->{state}  : '?',
+		);
+	}
+	return $ret;
+}
+
+1;

=== modified file 'lib/Monitor/t/Roles.t'
--- lib/Monitor/t/Roles.t	2009-02-05 08:43:52 +0000
+++ lib/Monitor/t/Roles.t	2010-03-09 10:25:26 +0000
@@ -55,7 +55,7 @@
 $roles->assign($role_writer, 'db1');
 is($roles->get_active_master(), 'db1', 'Active master after assigning writer role');
 
-$roles->clear_host_roles($roles->get_active_master());
+$roles->clear_roles($roles->get_active_master());
 is($roles->get_active_master(), '', 'No active master with active master host cleared');
 
 $roles->assign($role_writer, 'db2');
@@ -84,7 +84,7 @@
 is($roles->count_host_roles('db2'), 2, 'balance roles (role count db2)');
 
 $agents->{db2}->state('HARD_OFFLINE');
-$roles->clear_host_roles('db2');
+$roles->clear_roles('db2');
 $roles->process_orphans('exclusive');
 $roles->process_orphans('balanced');
 is($roles->count_host_roles('db1'), 4, 'process orphans assigns all orphaned roles');

=== modified file 'sbin/mmm_mond'
--- sbin/mmm_mond	2010-02-11 02:23:38 +0000
+++ sbin/mmm_mond	2010-03-09 10:25:26 +0000
@@ -72,11 +72,6 @@
 
 our $monitor	= new MMM::Monitor::Monitor::();
 
-if (!MMM::Monitor::NetworkChecker->initial_check()) {
-	LOGDIE "None of the 'ping_ips' could be reached during startup. Network seems to be down - mmm_mond will shutdown now.";
-}
-
-
 my $pidfilename = $config->{monitor}->{pid_path};
 my $pidfile = new MMM::Common::PidFile:: $pidfilename;
 
@@ -106,9 +101,9 @@
 $SIG{PIPE} = 'IGNORE';
 $SIG{CHLD} = \&ChildHandler;
 
-$monitor->init();
-
-$monitor->main();
+if ($monitor->init()) {
+	$monitor->main();
+}
 
 INFO 'END';
 exit(0);