#!/usr/bin/perl # # bb-3510fc.pl # # Big Brother watchdog for Sun Storage 3510 FC # # Author: Thomas Hoerndlein # e-Mail: bb@hoerndlein.de # Version: 1.5 # Date: 01.12.2005 # # # Remark V 1.5: Fixed bug which causes a red alarm if status for # diskslot is "Absent". # # Remark V 1.4: Release 1.4 - If your system supports reporting of # enclosure component status it will automatically # be monitored (fans, temparatures, voltages, etc). # # Remark V 1.3: Release 1.3 now runs with sccli versions 2.x as well # as 1.x versions (tested with sccli 1.6.2 and sccli 2.1.1). # In addition this version runs smoothly with a single # storage attached and multiples storages attached. # Previous versions only worked fine with MULTIPLE storages # attached. # # Remark V 1.2: Release 1.2 fixes just a presentational issue. On # some hardware configurations two coloured dots where # shown for each physical disk. Again thanks to Loris # Serena who pointed out this problem and did the # necessary testing. # # Remark V 1.1: This is a bugfix release of version 1.0. Solved problems: # In configurations with more than one device, the summary # color was not properly reported, if there was a disk # problem on any other device than the first one to be # checked. A similar problem affected the checking of # logical drives. # IT IS STRONGLY RECOMMENDED NOT TO USE VERSION 1.0 ANY # LONGER AND TO UPGRADE TO 1.1 INSTEAD!!! # Thanks to Loris Serena for pointing this out. # # Description: This script will do a health check of your Sun Storage # 3510 FC. # # Installation: 1. Make yourself a bit familiar with the sccli-tool # 2. Copy this file to $BBHOME/bb/ext # 3. chown root:others $BBHOME/bb/ext/bb-3510fc.pl # 4. chmod 4755 $BBHOME/bb/ext/bb-3510fc.pl => setuid bit # must be set! # 5. Set the correct values in the configuration section # 6. Add this script to ~bb/etc/bb-bbexttab # ########################################################################### # Section configuration ########################################################################### $ENV{PATH}="/usr/bin:/usr/sbin"; $BBPROG = "bb-3510fc.pl"; $BBHOME = "$ENV{BBHOME}"; $BBDISP = $ENV{BBDISP}; $MACHINE = $ENV{MACHINE}; $TEST="3510fc"; $debug = "N"; # # The hash below contains the number of the device (key) and the full path device name (value). # Calling 'sccli' from the command line will show this information: # %devicetocheck = ( 1 => '/dev/rdsk/c8t600C0FF00000000008656A5BD62C8000d0s2', 2 => '/dev/rdsk/c8t600C0FF0000000000865871B0230B100d0s2' ); # # The next hash holds the information about the disk status the system should have during # normal operation. Each disk in the 3510 FC Storage is uniquely identified by the following # convention: Device number -> Channel number -> Disk number # # %disktargetstat = ( # '--' => 'ONLINE', # '--' => 'STAND-BY', # .... # ); # # Some systems report only the primary channel, some report the secondary channel also # (this depends on your sccli version). For the latter one you need to enclose the # secondary channel in brackets (). Look for the Ch-column output of # >> sccli device show disks << # # Examples 1 - secondary channel IS NOT reported by your sccli version: # # %disktargetstat = ( # '2-2-0' => 'ONLINE', # '2-2-1' => 'STAND-BY', # .... # ); # # Examples 2 - secondary channel IS reported by your sccli version: # # %disktargetstat = ( # '2-2(3)-0' => 'ONLINE', # '2-2(3)-1' => 'STAND-BY', # .... # ); # %disktargetstat = ( '2-2(3)-6' => 'ONLINE', '2-2(3)-7' => 'ONLINE', '2-2(3)-8' => 'ONLINE', '2-2(3)-9' => 'ONLINE', '2-2(3)-10' => 'STAND-BY', '1-2(3)-6' => 'ONLINE', '1-2(3)-7' => 'ONLINE', '1-2(3)-8' => 'ONLINE', '1-2(3)-9' => 'ONLINE', '1-2(3)-10' => 'STAND-BY', ); $version = "1.5"; ########################################################################### # Section main ########################################################################### $ts = time(); $< = $>; # set real to effective uid @fullstat = (); $color = "red"; # check if devices are accessible: ($devcolor, @devstat) = _devcheck ( \%devicetocheck ); push fullstat, @devstat; push @fullstat, ""; push @fullstat, ""; _debug ( \$debug, \"DEBUG 1: \@fullstat after _devcheck():", \@fullstat ); _debug ( \$debug, \"DEBUG 1: color of test _devcheck():", \$devcolor ); # check disk status: $diskcolor = "green"; foreach $key (sort keys %devicetocheck) { @diskstat = (); ($testcolor, @diskstat) = _diskstat ( \$key, \%devicetocheck, \%disktargetstat ); push @fullstat, "Check of disks behind $devicetocheck{$key}:"; push @fullstat, ""; push @fullstat, @diskstat; push @fullstat, ""; push @fullstat, ""; if ($testcolor ne "green") { $diskcolor = "red"; } } _debug ( \$debug, \"DEBUG 2: \@fullstat after _diskstat():", \@fullstat ); _debug ( \$debug, \"DEBUG 2: color of test _diskstat():", \$diskcolor ); # now check status of logical drives: $logdrvcolor = "green"; foreach $key (sort keys %devicetocheck) { @diskstat = (); ($testcolor, @logdrvstat) = _logdrvstat ( \$devicetocheck{$key} ); push @fullstat, "Check of logical drives behind $devicetocheck{$key}:"; push @fullstat, ""; push @fullstat, @logdrvstat; push @fullstat, ""; push @fullstat, ""; if ($testcolor ne "green") { $logdrvcolor = "red"; } } _debug ( \$debug, \"DEBUG 3: \@fullstat after _logdrvstat():", \@fullstat ); _debug ( \$debug, \"DEBUG 3: color of test _logdrvstat():", \$logdrvcolor ); # now check enclosure status: $enclosurecolor = "green"; foreach $key (sort keys %devicetocheck) { @diskstat = (); ($testcolor, @enclosurestat) = _enclosurestat ( \$devicetocheck{$key} ); push @fullstat, "Check of enclosure behind $devicetocheck{$key}:"; push @fullstat, ""; push @fullstat, @enclosurestat; push @fullstat, ""; push @fullstat, ""; if ($testcolor ne "green") { $enclosurecolor = "red"; } } _debug ( \$debug, \"DEBUG 4: \@fullstat after _enclosurestat():", \@fullstat ); _debug ( \$debug, \"DEBUG 4: color of test _enclosurestat():", \$enclosurecolor ); # hier weiter proggn $color = "green" if ( $devcolor eq "green" && $diskcolor eq "green" && $logdrvcolor eq "green" && $enclosurecolor eq "green" ); _debug ( \$debug, \"DEBUG 5: \$color after all checks:", \$color ); $te = time(); $tdelta = $te - $ts; push @fullstat, "The job took $tdelta seconds to complete."; push @fullstat, "This is version $version of bb-3510fc.pl."; ########################################################################### # Section output ########################################################################### foreach (@fullstat) { $_ = "$_\n"; } @date=localtime(time()); $date=sprintf "%02i.%02i.20%02i %02i:%02i:%02i\n",$date[3],++$date[4],$date[5]-100,$date[2],$date[1],$date[0], ; $LINE = "status $MACHINE.$TEST $color $date @fullstat"; exec "$BBHOME/bin/bb", "$BBDISP", "$LINE"; #print "$BBHOME/bin/bb $BBDISP \"$LINE\""; ########################################################################### # Section functions ########################################################################### sub _debug { # call: _debug ( \$debugmode, \"comment", "\$var | \@array | \%hash ); my $refmode = $_[0]; my $refcomment = $_[1]; my $refvalue = $_[2]; my $mode = $$refmode; my $comment = $$refcomment; my $type; my $value; $type = ref $refvalue; if ( $mode eq "Y" ) { if ( $type eq "SCALAR" ) { $value = $$refvalue; print "$comment\n"; print "$value\n"; } elsif ( $type eq "ARRAY" ) { @value = @$refvalue; print "$comment\n"; foreach (@value) { print "$_\n"; } } elsif ( $type eq "HASH" ) { %value = %$refvalue; print "$comment\n"; foreach $key (sort keys %value) { print "$key => $value{$key}\n"; } } } } sub _devcheck { # call: ($color, @output) = _devcheck ( \%devtocheck ); my $refdevtocheck = $_[0]; my %devtocheck = %$refdevtocheck; my %dev_actual; my @select = (); my $devnr; my $rest; my $color = "green"; my @output = (); my $stor; @select = `sccli 2>&1 << EOF 1 q `; if (scalar @select <= 2) { splice (@select, -1); $select[0] =~ s/^sccli: selected device/ 1\./; push @output, "There is only one storage attached."; } else { splice (@select, 0, 2); splice (@select, -2); $stor = scalar @select; push @output, "There are $stor storages attached."; } chomp @select; # create a hash for all actual available devices foreach (@select) { s/^\s+//; # remove leading whitespace ($devnr, $rest) = split /\./; $_ = $rest; s/^\s+//; # remove leading whitespace s/\s+.*//; # eliminate obsolete info $dev_actual{$devnr} = $_; # print "\$dev_actual{$devnr}: $dev_actual{$devnr}\n"; } # Comparison between nominal value and actual value: foreach $key (sort keys %devtocheck) { if ( $devtocheck{$key} eq $dev_actual{$key} ) { push @output, "&green Device $devtocheck{$key} recognized."; } else { $color = "red"; push @output, "&red Device $devtocheck{$key} not recognized."; } } return $color, @output; } sub _logdrvstat { # call: ($color, @output) = _logdrvstat ( \$device ); my $refdevice = $_[0]; my $device = $$refdevice; my @logdrv = (); my $headline; my $underline; my $line; my ($ld, $ldid, $size, $assigned, $type, $disks, $spare, $failed, $status) = (); my $color = "green"; my @output = (); @logdrv=`sccli $device show logical-drives 2> /dev/null`; chomp @logdrv; $headline = shift @logdrv; $underline = shift @logdrv; push @output, "\t$headline"; push @output, "\t$underline"; foreach (@logdrv) { $line = $_; # save line for later use s/^\s//; # remove leading whitespace ($ld, $ldid, $size, $assigned, $type, $disks, $spare, $failed, $status) = split /\s+/; if ( $ldid ne 'Write-Policy' ) { # we don't want colors for write-policies if ( $status eq "Good" ) { push @output, "&green\t$line"; } else { $color = "red"; push @output, "&red\t$line"; } } else { push @output, "\t$line"; } } return $color, @output; } sub _diskstat { # call: ($color, @output) = _diskstat ( \$devnr, \%devtocheck, \%targetstat ); my $refdevnr = $_[0]; my $refdevtocheck = $_[1]; my $reftargetstat = $_[2]; my $devnr = $$refdevnr; my %devtocheck = %$refdevtocheck; my %targetstat = %$reftargetstat; my @disks = (); my $headline; my $underline; my $line; my ($ch, $id, $size, $speed, $ld, $status, $ids) = (); my $disknumber; my $color = "green"; my @output = (); my $device = $devtocheck{$devnr}; @disks=`sccli $device show disks 2> /dev/null`; chomp @disks; $headline = shift @disks; $underline = shift @disks; push @output, "\t$headline"; push @output, "\t$underline"; foreach (@disks) { $line = $_; # save line for later use s/^\s//; # remove leading whitespace ($ch, $id, $size, $speed, $ld, $status, $ids) = split /\s+/, $_, 7; $disknumber = "$devnr-$ch-$id"; if ($ch =~ /^[0-9]+\(?[0-9]*\)?$/ && $id =~ /^[0-9]+$/) { # eliminate double dot problem, accept redundant channel if ( $status eq $targetstat{$disknumber} ) { push @output, "&green\t$line"; } else { $color = "red"; push @output, "&red\t$line"; } } else { push @output, "\t$line"; } } return $color, @output; } sub _enclosurestat { # call: ($color, @output) = _enclosurestat ( \$device ); my $refdevice = $_[0]; my $device = $$refdevice; my @enclosure = (); my @chassis = (); my @component = (); my $headline; my $headline2; my $underline; my $line; my ($type, $unit, $stat, $frupn, $frusn, $adddata) = (); my $color = "green"; my @output = (); my $enclrep = 0; @enclosure=`sccli $device show enclosure-status 2>&1`; chomp @enclosure; $enclosure[0] =~ s/sccli: selected device //; # only cosmetic # determine if the system is able to report enclosure status # and split sccli output in chassis part and component part foreach (@enclosure) { $enclrep = 1 if /Enclosure Component Status:/; push @chassis, $_ if ($enclrep == 0); push @component, $_ if ($enclrep != 0); } if ($enclrep == 1) { $headline = shift @chassis; $headline2 = shift @chassis; $underline = shift @chassis; push @output, "\t$headline"; push @output, "\t$headline2"; push @output, "\t$underline"; foreach (@chassis) { $line = $_; # save line for later use s/^\s//; # remove leading whitespace if (/Status:/) { if ( /Status:\s+OK/ ) { push @output, "&green\t$line"; } else { $color = "red"; push @output, "&red\t$line"; } } else { push @output, "\t$line"; } } $headline = shift @component; $headline2 = shift @component; $underline = shift @component; push @output, "\t$headline"; push @output, "\t$headline2"; push @output, "\t$underline"; # remove element with highest index from @component if empty my $comp_maxind = scalar @component - 1; pop @component, $x if ($component[$comp_maxind] !~ /\w/); foreach (@component) { $line = $_; # save line for later use s/^\s+//; # remove leading whitespace ($type, $unit, $stat, $frupn, $frusn, $adddata) = split /\s+/; if ( $stat eq "OK" ) { push @output, "&green\t$line"; } elsif ( $stat eq "Absent" ) { push @output, "&clear\t$line"; } else { $color = "red"; push @output, "&red\t$line"; } } } else { push @output, "hmm..., seems that your system is not able to report an enclosure status for "; push @output, "device $device"; } return $color, @output; }