File check_pacemaker-node of Package monitoring-plugins-pacemaker-node

Overview Repositories Revisions Requests Users Attributes Meta

File check_pacemaker-node of Package monitoring-plugins-pacemaker-node

#!/usr/bin/perl -w
#
# About:        A script for monitoring Linux HA status, to be used as a
#               NRPE Plugin for pacemaker.
# Author:       Martin  Caj mcaj@suse.cz
#               Ruediger Oertel ro@suse.de
# Last change:  2011-10-27
#               2017-03-22
# TODO: 	rewrite res-check to awk ?
#		use awk to check/print more about cluster status
#               add check for sudo 
#		integrate stonith devices check 
# 		
#############################################################################

use strict;
use Getopt::Std;
use Data::Dumper;
#############
# Variables #
#############

my $NODE_NAME=`uname -n|tr "[:upper:]" "[:lower:]"`;
chomp($NODE_NAME);
my $CRM_MON="/usr/sbin/crm_mon";
my $ADMIN_EMAIL="mcaj\@suse.cz";

###################################
# Resources and DRBD are options :#
###################################
# you should defive them via /etc/nagios/nrpe.cfg
# eg:

#command[check_pacemaker-nod]=/usr/lib/nagios/plugins/check_pacemaker-nod -r drbd_example fs_example extIP intIP vpn_exemple route_EXT

######################
# Nagios Error Codes #
######################

my $OK="0"; # = OK
my $WARNING="1"; # = Warning
my $CRITICAL="2"; # = Fatal Error
my $UNKNOWN="3"; # UNKNOWN Error

my $retval;
my $result;
my $smap = ["OK","WARNING","CRITICAL","UNKNOWN"];

#####################
# function def part.#
#####################

#---------------------------------------------------------------------------------------------
# usage:
# Printing help if the script is run without any options
# the node is: 
#               online  - script can continue
#               offline - HA service isn't running - exit CRITICAL
#               standby = the server is in standby mode no more check are need. exit OK 
#---------------------------------------------------------------------------------------------

sub usage {
	my ($retval) = @_;
	print "\n";
	print "This is a Nagios Check for Pacemaker Status.\n";
        print "It has been designed for checking the cluster via NRPE directly on the machine,\n";
	print "where the cluster (e.g.: openais) is running\n";
	print "For this check, the nagios user has to be allow run /usr/sbin/crm_mon as sudo.\n";
	print "Check setting sudo rigths via visudo command.\n";
	print "\n";
        print " usage:\n";
        print "       -h : help\n";
        print "       -p : print list of resources for check\n";
	print "       -r : run check\n";
	print "       -d : check deciding if the node is master or slave (comma separated list)\n";
	print "            all in this group should run on the same node\n";
	print "       -m : resources that should be running on the master (comma separated list)\n";
	print "       -s : resources that should be running on the slave (comma separated list)\n";
	print "       -b : resources that should be running on the both (comma separated list)\n";
	print "\n";
	print "this is a example for an entry in the nagios nrpe file /etc/nagios/nrpe.cfg\n";
	print "command[check_pacemaker-node]=/usr/lib/nagios/plugins/check_pacemaker-node -r -d drbd_example -m fs_example,extIP,intIP,vpn_exemple,route_EXT\n";
	print "\n";
        print "If you have any questins please contact local admins on email \"$ADMIN_EMAIL\"\n";
        print "\n";
        exit $retval;
}

sub print_to_log {
	my ($severity,$message) = @_;
	push @{$result->{$severity}}, $message;
}

sub print_from_short_log {
	print join(", ", @{$result->{'short'}})."\n";
}

sub print_from_log {
	my ($severity) = @_;
	for (@{$result->{$severity}}) {
	    print "$smap->[$severity]: $_\n";
	}
}

#---------------------------------------------------------------------------------------------
# usage:
# Testing node itself
# there are three possible cases
# the node is: 
# 		online  - script can continue
# 		offline - HA service isn't running - exit CRITICAL
#		standby = the server is in standby mode no more check are need. it can continue
#-----------------------------------------------------------------------------------------------

sub check_node_online {
	my ($crmstat,$nodename) = @_;
	my $res_status;
	my $retval;
	if (grep { $_ eq $nodename } @{$crmstat->{'Online'}}) {
		$res_status = "Online";
		$retval = $OK;
	} elsif (grep { $_ eq $nodename } @{$crmstat->{'Offline'}}) {
		$res_status = "Offline";
		$retval = $CRITICAL;
	} elsif (grep { $_ eq $nodename } @{$crmstat->{'Standby'}}) {
		$res_status = "Standby";
		$retval = $OK;
	} else {
		$res_status = "Unknown";
		$retval = $UNKNOWN;
	}
	
	print_to_log($retval,"The node \"$nodename\" has status \"$res_status\".");
	print_to_log($retval," Please check the status ASAP!") if $retval;
	print_to_log("short", "$nodename:$res_status");
		
	return ($retval, $res_status);
}

#---------------------------------------------------------------------------------------------
# usage:
# Testiing it there aren't any failed resources.
# During migtation process might some resource failed, but when the migration is done 
# there should be non. it the script find a failed resource it wait 20s and try it again
# then it ends with CRITICAL exit.
#---------------------------------------------------------------------------------------------

sub check_failed {
	my ($crmstat) = @_;
	my $retval = $OK;
	if ($crmstat->{'Failed'}) {
		$retval = $WARNING;
		print_to_log($retval, "here are some failed actions, please check crm ASAP!");
		print_to_log($retval, "failed actions are:");
		my $flist = join(",",@{$crmstat->{'Failed'}});
		print_to_log($retval,$flist);
		print_to_log("short", "Failed:$flist");
	}
	unless ($crmstat->{'Online'} || $crmstat->{'Offline'} || $crmstat->{'Standby'}) {
		$retval = $UNKNOWN;
		print_to_log($retval, "There is something wrong");
		print_to_log($retval, "Could not parse crm_mon output");
		print_to_log($retval, "Please check if nagios user is allowed to run");
		print_to_log($retval, "\"/usr/sbin/crm_mon\" via sudo");
	}
	return $retval;
}

#---------------------------------------------------------------------------------------------
# usage:
# Testing Resources
# if you have simmilar resource name, you can use exclude via command grep -v
# example is us in check_res
#---------------------------------------------------------------------------------------------

sub check_res {
	my ($crmstat,$nodename,$resname) = @_;
	my $retval;
	if (grep {$_ eq $nodename} @{$crmstat->{'resource'}->{$resname}->{'Started'}}) {
		$retval = $OK;
		print_to_log($retval, "The resource \"$resname\" is fine");
	} elsif (grep {$_ eq $nodename} @{$crmstat->{'Clone Set'}->{$resname}->{'Started'}}) {
		$retval = $OK;
		print_to_log($retval, "The resource \"$resname\" is fine");
	} else {
		$retval = $CRITICAL;
		print_to_log($retval, "The resource \"$resname\" is not running on this node");
		print_to_log("short", "missing:$resname");
	}
	return $retval;
}

#---------------------------------------------------------------------------------------------
# usage:
# Testing drbd status
# drbd Might be in the mode :
# Masters:
# Slaves: 
# Stopped: if the server is in the snadby mode - exit OK else CTITICAL !
# the master node has usally runnig all services on it self.
sub check_ms {
	my ($crmstat,$nodename,$res_ms) = @_;
	my $retval;
	for my $resource (@{$res_ms}) {
	    my $rstate; 
	    for my $state ("Masters","Slaves","Stopped") {
		$rstate = $state if grep { $_ eq $nodename } @{$crmstat->{'Master/Slave Set'}->{$resource}->{$state}};
		next unless $rstate;
		if ($retval) {
		    $retval = "MISMATCH" unless $retval eq $rstate;
		} else {
		    $retval = $rstate;
		}
	    }
	    if ($retval eq "MISMATCH") {
		print_to_log($WARNING,"M/S resource $resource status $rstate");
	    } else {
		print_to_log($OK,"M/S resource $resource status $rstate");
	    }
	    unless ($rstate) {
		print_to_log($CRITICAL,"M/S resource $resource not found");
		$retval = $CRITICAL;
	    }
	}
	return $retval;
}

################
# running part:#
################

usage (1) unless $#ARGV >= 0;

our($opt_h,$opt_r,$opt_p,$opt_d,$opt_m,$opt_s,$opt_b);
usage unless getopts('hrpd:m:s:b:');

usage(0) if $opt_h;
usage(1) if @ARGV;

my ($res_ms,$res_on_master,$res_on_slave,$res_on_both);
@{$res_ms} = split(",",$opt_d || "");
@{$res_on_master} = split(",",$opt_m || "");
@{$res_on_slave} = split(",",$opt_s || "");
@{$res_on_both} = split(",",$opt_b || "");

if ($opt_p) {
	print "The resources for checking are:\n";
	for my $i (0..scalar(@{$res_ms})-1) {
	    print "Master/Slave resource $res_ms->[$i],\n";
	}
	for my $i (0..scalar(@{$res_on_master})-1) {
	    print "Master should have running $res_on_master->[$i],\n";
	}
	for my $i (0..scalar(@{$res_on_slave})-1) {
	    print "Slave should have running $res_on_slave->[$i],\n";
	}
	for my $i (0..scalar(@{$res_on_both})-1) {
	    print "Both should have running $res_on_both->[$i],\n";
	}
	print "\n";
	print "The name of this node is \"$NODE_NAME\".\n";
	print "Provides a summary of cluster's current state via \"sudo $CRM_MON\".\n";
	print "local admin contact is \"$ADMIN_EMAIL\".\n";
	print "\n";
	print "runng one short cluster status....\n";
	system("sudo $CRM_MON -1");
	print "done\n";
	exit $UNKNOWN;
}

if ($opt_r) {
	open (CRM, "sudo $CRM_MON -1 |");
	my @CRMOUT = <CRM>;
	chomp(@CRMOUT);
	close (CRM);
	@CRMOUT = grep { !/^$/ } @CRMOUT;
	my $crmstat;
	while ($_ = shift @CRMOUT) {
	    if (/^ /) {
		if (/\[.*\]/) {
		    if (/^\s+(.* Set): (.*) \[(.*)\]/) {
			my $type = $1;
			my $name = $3;
			while ($_ = shift(@CRMOUT), /^\s+\s+/) {
			    push @{$crmstat->{$type}->{$name}->{$1}}, split('\s+',$2) if /^\s+(.*):\s+\[\s+(.*)\s+\]/;
			}
			unshift @CRMOUT, $_;
		    }
		} else {
		    push @{$crmstat->{'resource'}->{$1}->{$3}}, split('\s+',$4) if /^\s+(.*)\s+(.*)\s+(St.*)\s+(.*)$/;
		    push @{$crmstat->{'resource'}->{$2}->{$3}}, split('\s+',$4) if /^\s+(.*)\s+$(.*)$:\s+(St.*)\s+(.*)$/;
		}
	    } else {
		if (/^Online:/ || /^Offline:/) {
		    push @{$crmstat->{$1}}, split('\s+',$2) if /^(.*):\s+\[ (.*) \].*/;
		} elsif (/^Node/) {
		    push @{$crmstat->{Standby}}, split('\s+',$2) if /^(.*):\s+\[ (.*) \].*/;
		} elsif (/^Failed/) {
		    push @{$crmstat->{Failed}}, split('\s+',$2) if /^(.*):\s+(.*)/;
		}
	    }
	}

# print Dumper($crmstat);

# the first test if node is online/offline/standby
	my $res_status;
	($retval, $res_status) = check_node_online($crmstat,$NODE_NAME);

#test if all resvices are running well.
	$retval |= check_failed($crmstat);

#-------------------------------------
	# DRBD check MASTER/SLAVE. 
	# res 1-12 are master resources
	# res 13 is slave resources
	# res 14 might be run on both. 
	# -----------------------------------

if ($res_ms) {
		my $ms_state = check_ms($crmstat,$NODE_NAME,$res_ms);
		if ($ms_state eq "Masters") {
			print_to_log($OK, "Master/Slave resources running as Master");
			print_to_log("short", "master/slave:".join(",",@{$res_ms}).":master");
			for my $i (0..scalar(@{$res_on_master})-1) {
				if ($res_on_master->[$i]) {
					$retval |= check_res($crmstat,$NODE_NAME,$res_on_master->[$i]);
				}
			}
			for my $i (0..scalar(@{$res_on_both})-1) {
				if ($res_on_both->[$i]) {
					$retval |= check_res($crmstat,$NODE_NAME,$res_on_both->[$i]);
				}
			}
			if ($retval & $CRITICAL) {
			    $retval = $CRITICAL;
			} elsif ($retval & $WARNING) {
			    $retval = $WARNING;
			} else {
			    $retval = $OK;
			    print_to_log($retval, "all resources on the Master node \"$NODE_NAME\" are fine ;-)");
			}
		} elsif ($ms_state eq "Slaves") {
			print_to_log($OK, "Master/Slave resources  running as Slave");
			print_to_log("short", "master/slave:".join(",",@{$res_ms}).":slave");
			for my $i (0..scalar(@{$res_on_slave})-1) {
				if ($res_on_slave->[$i]) {
					$retval |= check_res($crmstat,$NODE_NAME,$res_on_slave->[$i]);
				}
			}
			for my $i (0..scalar(@{$res_on_both})-1) {
				if ($res_on_both->[$i]) {
					$retval |= check_res($crmstat,$NODE_NAME,$res_on_both->[$i]);
				}
			}
			if ($retval & $CRITICAL) {
			    $retval = $CRITICAL;
			} elsif ($retval & $WARNING) {
			    $retval = $WARNING;
			} else {
			    $retval = $OK;
			    print_to_log($retval, "all resources on the Slave node \"$NODE_NAME\" are fine ;-)");
			}
		} elsif ($ms_state eq "Stopped") {
			print_to_log($OK, "Master/Slave resources stopped on this node");
			print_to_log("short", "master/slave:".join(",",@{$res_ms}).":stopped");
			print_to_log($OK, "No resources are running on this node \"$NODE_NAME\"");
			if ($res_status eq "Standby") {
				$retval = $OK;
				print_to_log($retval, "Status: Current server \"$NODE_NAME\" is in \"$res_status\" mode.");
				print_to_log($retval, "To switch it to online mode, you have to do this via crm.");
			} else {
				$retval = $WARNING;
				print_to_log($retval, "Master/Slave resources stopped but server \"$NODE_NAME\" is not in standby mode:");
				print_to_log($retval, "The server status is: \"$res_status\"");
				print_to_log($retval, "There is something wrong. Check the server \"$NODE_NAME\" ASAP!");
			}
		} else {
			$retval = $CRITICAL;
			print_to_log($retval, "Master/Slave resources not running correctly, it has status \"$ms_state\".");
			print_to_log($retval, "There is something wrong. Check the server \"$NODE_NAME\" ASAP!");
			print_to_log("short", "master/slave:mismatched");
		}
	} else {
		$retval = $UNKNOWN;
		print_to_log($retval, "Master/Slave resources not defined. check your nrpe.conf file.");
	}

}

print $smap->[$retval].": ";
print_from_short_log();

for my $severity (3,2,1,0) {
    print_from_log($severity);
}

exit $retval;
# end.

Places

File check_pacemaker-node of Package monitoring-plugins-pacemaker-node

Places