Verifying Firmware (OBP) on Oracle (Sun Microsystems) Hardware

To verify the firmware of a Sun Microsystems server use one of the following options:

At the OBP, use the '.version' command:

ok .version
Firmware CORE Release 1.0.18 created 2002/5/23 18:22
Release 4.0 Version 18 created 2002/05/23 18:22
cPOST version 1.0.18 created 2002/5/23
CORE 1.0.18 2002/05/23 18:22
ok

When running Solaris, use the prtconf(1M) command:

# prtconf -V
OBP 4.0.18 2002/05/23 18:22
#

x86 Hardware RAID Traps with BS_RAID_CHK

The following script is designed to run on Solaris x86 or Redhat systems with LSI and Adaptec hardware controllers.  It will check for degraded states of those controllers using mpt-status, arccon or raidctl depending on hardware vendor and OS.   If a degraded state is found it will send an snmptrap to your snmptrap collector.

#!/usr/bin/perl
#################################################################
# Script checks status of hardware raid on X86 hardware        #
# Supported OS's: Solaris x86, Redhat                #
# Supported Controller's: LSI & Adaptec                #
# Note: Requires mpt-status for LSI                #
# Note: Requires arccon for adaptec                #
# Note: Requires raidctl for Solaris x86            #
# Sends trap if degraded state                    #
#################################################################
use strict;
my $prefix = "bs_raid_chk";
my $servicechk = "unix_traps";
my $community = "asdpublic";
my $manager = "10.66.65.23";
my $raidctl = '/usr/sbin/raidctl';
my $mptstatus = '/usr/sbin/mpt-status';
my $arcconf = '/usr/StorMan/arcconf';
my (@components,@command);
my ($num,$status,$sendtrap,$volume);
### If Solaris System use this check ###
if (`uname -a` =~ /SunOS/) {
    if ( -e $raidctl) {
        @command = `$raidctl -S`;
        foreach (@command) {
            chomp();
            @components = split();
            $num = $components[1] + 2;
            $status = "$components[$#components] - Controller: $components[0], RAID: $components[$num], Number of Disks: $components[1]\n";
            if ($_ =~ /DEGRADED/) {
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                system "/bin/touch /tmp/$prefix.CRITICAL";
                system "/usr/sfw/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1005 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                        exit;
            } elsif ($_ =~ /SYNC/) {
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                                system "/bin/touch /tmp/$prefix.WARNING";
                system "/usr/sfw/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1006 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                        exit;
            } elsif ($_ =~ /OPTIMAL/) {
                if ( !-e "/tmp/$prefix.OK" ) {
                    system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                    system "/bin/touch /tmp/$prefix.OK";
                    system "/usr/sfw/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1007 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                }
            }
        }
    }
}
### If Linux system use this check  ###
if (`uname -a` =~ /Linux/) {
    # If system has LSI controller, then mptstatus should be installed
    if (-e $mptstatus) {
        my $modstatus = `/sbin/lsmod |grep mptctl|wc -l`;
        chomp($modstatus);
        if ($modstatus eq "0") {
            my $modload = `/sbin/modprobe mptctl`;
            $modstatus = `/sbin/lsmod |grep mptctl|wc -l`;
            chomp($modstatus);
            if ($modstatus eq "0") { print "ABORT: Failed to load mptctl module.\n";exit;}
        }
        my $controller = `$mptstatus -p -s|grep Found`;
        chomp($controller);
        my ($id,$junk) = split(/,/,$controller);
        $id =~ s/Found SCSI id=//g;
        @command = `$mptstatus -i $id -s`;
        $status="";
        foreach (@command) {
            chomp();
            $status = "$status $_";   
        }
        $status = "$status";
        #print "$status\n";
        foreach (@command) {
            chomp();
            if ( $_ =~ /DEGRADED/ ) {
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                system "/bin/touch /tmp/$prefix.CRITICAL";   
                system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1005 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                exit;
            } elsif ($_ =~ /SYNC/ ) {
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                system "/bin/touch /tmp/$prefix.WARNING";
                system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1006 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                exit;
            } elsif ($_ =~ /OPTIMAL/ ) {
                if ( !-e "/tmp/$prefix.OK" ) {
                    system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                    system "/bin/touch /tmp/$prefix.OK";
                    system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1007 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                }
            }
        }
    }
    # if system has Adaptec controller then arcconf should be installed
    if ( -e $arcconf ) {
        @command = `$arcconf getconfig 1|grep Status|grep :`;
        foreach (@command) {
            if (( $_ =~ /Controller Status/ ) && ($_ !~ /Optimal/ )) {
                $status = "Controller not optimal";
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                system "/bin/touch /tmp/$prefix.CRITICAL";
                system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1005 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                exit;
            }
            if (( $_ =~ /  Status  / ) && ($_ !~ /Optimal/ )) {
                $status = "Battery not optimal";
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                system "/bin/touch /tmp/$prefix.WARNING";
                system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1006 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                exit;
            }
            if (( $_ =~ /Status of logical device/) && ($_ !~ /Optimal/ )) {
                $status = "Logical HW RAID Volume not optimal";
                system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                system "/bin/touch /tmp/$prefix.CRITICAL";
                system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1005 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                exit;
            }
            if ( $_ =~ /Optimal/ ) {
                if ( !-e "/tmp/$prefix.OK" ) {
                    $status = "Hardware RAID - OK";
                    system "/bin/rm /tmp/$prefix.* >/dev/null 2>&1";
                    system "/bin/touch /tmp/$prefix.OK";
                    system "/usr/bin/snmptrap -v 2c -c $community $manager '' .1.3.6.1.4.1.11.2.17.1.0.1007 .1.3.6.1.4.1.11.2.17 s \"$servicechk\" .1.3.6.1.4.1.11.2.17 s \"$status\"";
                }
            }
        }
    }   
}
exit; 

Persistantly Bind Tape Devices in Solaris via Perl

The following script will look for fiber channel tape devices and then configure the devlinks.tab file with the appropriate information so the tape drives will persistently bind to the same device across reboots on a Solaris server.   This script was tested on Solaris 10.

#!/usr/bin/perl
use strict;
my($junk,$path,$devices,$dev,$file);
my(@devices,@file);
my $date = `date +%m%d%Y`;
$file = `/usr/bin/cp /etc/devlink.tab /etc/devlink.tab.$date`;
@file = `cat /etc/devlink.tab`;
@file = grep !/type=ddi_byte:tape/, @file;
open (FILE,">/etc/devlink.tab.new");
print FILE @file;
close (FILE);
 
@devices = `ls -l /dev/rmt/*cbn|awk {'print \$9 \$11'}`;
open (FILE,">>/etc/devlink.tab.new");
foreach $devices (@devices) {
        chomp($devices);
        ($dev,$path) = split(/\.\.\/\.\./,$devices);
        $dev =~ s/cbn//g;
        $dev =~ s/\/dev\/rmt\///g;
        $path =~ s/:cbn//g;
        ($junk,$path) = split(/st\@/,$path);
        print FILE "type=ddi_byte:tape;addr=$path;\trmt/$dev\\M0\n";
}
close (FILE);
$file = `/usr/bin/mv /etc/devlink.tab.new /etc/devlink.tab`;
exit;

Solaris LUN Online Report

If you are using fiber channel storage with Solaris in a multipath configuration, sometimes before fabric maintenance or array maintenance you might want to check and confirm the status of all the paths on the Solaris client.   The following script utilizing luxadm will report on the status of each path for a fiber channel device.

#!/bin/perl
@luns = `/usr/sbin/luxadm probe | grep Logical | sed -e 's/.*\://g'|grep rdsk`;
foreach $lun (@luns) {
        chomp($lun);
        $lun2 = $lun;
        $lun2 =~ s/\/dev\/rdsk\///g;
        print "Disk:$lun2\t";
        @luxadm = `/usr/sbin/luxadm display $lun`;
        $pathcount = 0;
        foreach $luxadm (@luxadm) {
                chomp($luxadm);
                if ($luxadm =~ /State/) {
                        $luxadm =~ s/State//g;
                        $luxadm =~ s/^\s+//;
                        print "Path$pathcount:$luxadm\t";
                        $pathcount++;
                }
        }
        print "\n";
}

The output from the script will look something like the output below:

#perl pathfinder.pl
Disk:c6t60060E80054337000000433700000526d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE
Disk:c6t60060E80054337000000433700000527d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE
Disk:c6t60060E80054337000000433700000301d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE
Disk:c6t60060E80054337000000433700000300d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE
Disk:c6t60060E80054337000000433700000278d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE
Disk:c6t60060E80054337000000433700000277d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE
Disk:c6t60060E80054337000000433700000275d0s2    Path0:ONLINE    Path1:ONLINE    Path2:ONLINE    Path3:ONLINE

Cleanup Shared Memory Segments Solaris

If you have ever used an application in Solaris that uses shared memory and that application has a tendency to not cleanup those memory segments properly on shutdown (SAP comes to mind)  then this little Perl script is what you have been waiting for.

All this script does is take certain field output from the ipcs command and then iterate through it to determine if the memory is still actively in use by a process or if it it can safely be purged.   I recommended testing this out with the $memclean line commented out to gain a good understanding before you remove the comment and allow the cleanup (#$memclean = `ipcrm -m $shmem`;).  This script was tested on Solaris 10.

#!/usr/bin/perl
@sms = `ipcs -pm|grep "^m"|awk {'print \$2":"\$7":"\$8'}`;
foreach $sms (@sms) {
        chomp($sms);
       ($shmem,$cpid,$lpid) = split(/:/,$sms);
        $cpids=` ps -ef|grep $cpid|grep -v grep >/dev/null 2>&1;echo \$?`;
        $lpids=` ps -ef|grep $lpid|grep -v grep >/dev/null 2>&1;echo \$?`;
        chomp($cpids,$lpids);
        if (($cpids eq "1") && ($lpids eq "1")) {
                $message = "Memory can be reclaimed";
                #$memclean = `ipcrm -m $shmem`;
        } else {
                $message = "Memory active";
        }
        print "$shmem - $cpid - $lpid - $cpids - $lpids - $message\n";
}

The output from the script will look similar to the following:

# perl mem_clean.pl
587203562 - 10885 - 17891 - 0 - 0 - Memory active
922746991 - 9728 - 10885 - 0 - 0 - Memory active
150995435 - 9728 - 10885 - 0 - 0 - Memory active
150995432 - 9728 - 17891 - 0 - 0 - Memory active
150995398 - 17421 - 17891 - 1 - 0 - Memory active
150995396 - 13421 - 13891 - 1 - 1 - Memory can be reclaimed
150995387 - 9728 - 10885 - 0 - 0 - Memory active
150995382 - 9728 - 10885 - 0 - 0 - Memory active
150995380 - 9728 - 10885 - 0 - 0 - Memory active
150995379 - 9728 - 10885 - 0 - 0 - Memory active
150995377 - 9728 - 17891 - 0 - 0 - Memory active
150995374 - 9728 - 17891 - 0 - 0 - Memory active
150995371 - 9727 - 10886 - 1 - 0 - Memory active
117440821 - 9728 - 10885 - 0 - 0 - Memory active
117440819 - 9728 - 10885 - 0 - 0 - Memory active

Determine Core Count on Solaris 10 Hosts

This question always comes up time and time again. How does one figure out how to get the correct core count on a Solaris 10 host?  Below is the one line answer:

echo "`hostname` has `kstat cpu_info |grep core_id|uniq|wc -l` cores"

Solaris 10 & What Process is Using Port

The other day I had a request to find out what process was using a specific port on a Solaris 10 server.  I came up with this little gem to do the work and provide the PID of the process using the port.

get_port.sh

#!/bin/bash

if [ $# -lt 1 ]
then
echo "Usage: ./get_port.sh port#"
exit
fi
echo "Searching for PID using port... "
for i in `ls /proc`
do
pfiles $i | grep AF_INET | grep $1
if [ $? -eq 0 ]
then
echo The port is in use by PID: $i
fi
done

Sun Cluster 3.2 & SCSI Reservation Issues

If you have worked with luns and Sun Cluster 3.2, you may have discovered that if you ever want to remove a lun from a system, it may not be possible because of the scsi3 reservation that Sun Cluster places on the disks.  The example scenario below walks you through how to overcome this issue and proceed as though Sun Cluster is not even installed.

Example:  We had a 100GB lun off of a Hitachi disk array that we were using in a metaset that was controlled by Sun Cluster. We had removed the resource from the Sun Cluster configuration and removed the device with configadm/devfsadm, however when the storage admin attempted to remove the lun id from the Hitachi array zone, the Hitach array indicated the lun was still in use.  From the Solaris server side, it did not appear to be in use, however Sun Cluster has set the scsi3 reservations on the disk.

Clearing the Sun Cluster scsi reservation steps:

1) Determine what DID device the lun is mapped to using /usr/cluster/bin/scdidadm -L
2) Disable failfast on the DID device using /usr/cluster/lib/sc/scsi -c disfailfast -d /dev/did/rdsk/DID
3) Release the DID device using  /usr/cluster/lib/sc/scsi -c release -d /dev/did/rdsk/DID
4) Scrub the reserve keys from the DID device using  /usr/cluster/lib/sc/scsi -c scrub -d /dev/did/rdsk/DID
5) Confirm reserve keys are removed using /usr/cluster/lib/sc/scsi -c inkeys -d /dev/did/rdsk/DID
6) Remove lun from zone on machine or whatever procedure you were trying to complete.

Configuring Persistent Bindings on Solaris 10

If you have tape devices attached to your Solaris 10 host and you often find that after a reboot of the host, the tape devices are no longer in the same order they were before, you can use the following Perl script to configure the /etc/devlink.tab file to make the tape devices persist.  Script is below:

#!/usr/bin/perl
#################################################################
# This script maps fiber attached tape drives to persistently   #
# bind to the same device across reboots.                       #
# (C) 2011 Benjamin Schmaus                                     #
#################################################################
use strict;
my($junk,$path,$devices,$dev,$file);
my(@devices,@file);
my $date = `date +%m%d%Y`;
$file = `/usr/bin/cp /etc/devlink.tab /etc/devlink.tab.$date`;
@file = `cat /etc/devlink.tab`;
@file = grep !/type=ddi_byte:tape/, @file;
open (FILE,">/etc/devlink.tab.new");
print FILE @file;
close (FILE);
@devices = `ls -l /dev/rmt/*cbn|awk {'print \$9 \$11'}`;
open (FILE,">>/etc/devlink.tab.new");
foreach $devices (@devices) {
                chomp($devices);
                ($dev,$path) = split(/\.\.\/\.\./,$devices);
                $dev =~ s/cbn//g;
                $dev =~ s/\/dev\/rmt\///g;
                $path =~ s/:cbn//g;
                ($junk,$path) = split(/st\@/,$path);
                print FILE "type=ddi_byte:tape;addr=$path;\trmt/$dev\\M0\n";
}
close (FILE);
$file = `/usr/bin/mv /etc/devlink.tab.new /etc/devlink.tab`;      
exit

Mapping Global Zone Name Inside Solaris 10 Zone

Sometimes when I am inside of a zone I want to know what global zone its attached to.  By default this was not visible but with a little effort below we can set it up so the global zone is visible within the zone.

Log into global zone where local zone(s) reside.

Using zonecfg, configure the following for each zone:
# zonecfg -z (zone)
add fs
set type=lofs
set options=ro
set special=/etc/nodename
set dir=/etc/globalname
end
verify
commit
exit

Create mount point within local zone directory structure:
# touch /zones/(zone)/root/etc/globalname

Mount lofs file system manually:
# mount -F lofs -o ro /etc/nodename /zones/(zone)/root/etc/globalname (or path where the root of the zone resides)

Confirm local zone can access file:
# zlogin (zone) cat /etc/globalname

Change /tmp size on Solaris 10 Zone

This blog describes the steps on how to change the tmp size on an existing Solaris 10 zone.

First log into global zone where local zone(s) reside using zlogin:

# zlogin –z (zone)

As root edit the /etc/vfstab:

# vi /etc/vfstab

Find the line in vfstab for the /tmp filesystem:

swap - /tmp tmpfs - yes size=512mb (this could be any value, 512 is example)

Change the value of size=512mb to the requested value (in MB):

swap - /tmp tmpfs - yes size=2048mb

Save the vfstab and exit back to the global zone.

To make the changes take effect the zone must be stopped and booted:

# zoneadm –z (zone) halt

# zoneadm –z (zone) boot

  Log back into local zone and confirm changes by reviewing df –h output:

# zlogin –z

# df –h|grep /tmp

swap                   2.0G     0K   2.0G     0%    /tmp

Script to Check StorEdge 3510 Disk Status

This little script will run the sccli utility and report out the disk status from the Sun Microsystems StorEdge 3510.

#!/usr/local/bin/zsh
#################################################################
# This script checks 3510 status                                #
#################################################################
EMAILADDR="sysadmin@software.umn.edu"
FAILSTAT="0"
FAILSTAT1="0"
for DISKS in `/usr/local/sccli/sbin/sccli -l|/usr/local/bin/cut -d" " -f1 -`
do
SCCLI=`/usr/local/sccli/sbin/sccli $DISKS show logical-drives 2>&1 > /tmp/lds.out`
LDS=`/usr/local/bin/cat /tmp/lds.out 2>&1|grep ld|grep -v Good`
if [ $? = 1 ]; then
FAILSTAT="0"
else
FAILSTAT1="1"
fi
SCCLI=`/usr/local/sccli/sbin/sccli $DISKS show enclosure-status 2>&1 > /tmp/sccli.out`
for TESTS in Topology Fan PS Temp Voltage DiskSlot
do
CHECK=`/usr/local/bin/cat /tmp/sccli.out 2>&1|grep $TESTS|grep -v OK`
if [ $? = 1 ]; then
FAILSTAT="0"
else
FAILSTAT1="1"
fi
done
if [ $FAILSTAT1 = 1 ]; then
MERGE=`/usr/local/bin/cat /tmp/sccli.out>/tmp/report.out`
MERGE=`echo>>/tmp/report.out`
MERGE=`/usr/local/bin/cat /tmp/lds.out>>/tmp/report.out`
SYSTEM=`hostname`
SUBJECTLINE="StoreEdge Error -- HOST: $SYSTEM DISK: $DISKS"
/bin/mailx -s $SUBJECTLINE $EMAILADDR < /tmp/report.out
fi
done
exit

Script to Check Solaris DiskSuite States

This Kourne shell script will check for any errors on metadevices with Disksuite on Solaris.

#!/bin/ksh
#########################################################################
# check_disks - check metadevices for errors and alert the user. #
# USAGE: check_disks [ -m (address) ] #
#########################################################################
TZ=CST5CDT
SUBJECT="MetaDisk Errors on `uname -n`"
GREP_CMD="/bin/egrep"
GREP_ARG="-s"
METASTAT_CMD=/usr/opt/SUNWmd/sbin/metastat
METADB_CMD=/usr/opt/SUNWmd/sbin/metadb
DISKSUITE_PRESENT=1
MAIL_OUTPUT=''
MAIL_RECIP=''
DEFAULT_MAIL_RECIP=''
OUTPUT_MSG=""
RESULT=''
OUTPUT_CODE=0
#########################################################################
# FUNCTION DEFINITIONS #
#########################################################################
function error_out {
   RETURN_CODE=$2
   ERROR_MSG=$1
   print - $ERROR_MSG
   exit $RETURN_CODE
}
#########################################################################
# MAIN PROGRAM #
#########################################################################
while getopts :m: c
do
case $c in
   m ) MAIL_OUTPUT="YES" # any non-null string will do
   MAIL_RECIP=$OPTARG
   if [[ -z $MAIL_RECIP ]]
   then
      MAIL_RECIP=$DEFAULT_MAIL_RECIP
   fi
  ;;
  ?) error_out "Usage: check_disks [ -m mail_addr ]" 1
  ;;
esac
done
if [ ! -x $GREP_CMD ]
then
   GREP_CMD="/usr/xpg4/bin/egrep"
fi

if [ ! -x $GREP_CMD ]
then
   print "ERROR: egrep not executable or not found"
   exit 1
fi
PATH=$PATH:/usr/sbin
METASTAT_BIN=$(/bin/which metastat)
if [ ! -x $METASTAT_CMD ]
   then
   if [ -x $METASTAT_BIN ]
   then
      METASTAT_CMD=$METASTAT_BIN
      DISKSUITE_PRESENT=1
   else
      DISKSUITE_PRESENT=0
   fi
fi
METADB_BIN=$(/bin/which metadb)
if [ ! -x $METADB_CMD ]
then
   if [ -x $METADB_BIN ]
   then
      METADB_CMD=$METADB_BIN
      DISKSUITE_PRESENT=1
   else
      DISKSUITE_PRESENT=0
   fi
fi
if [[ $DISKSUITE_PRESENT -eq 0 ]]
then
   error_out "DiskSuite has been found." 3
fi
#################################################################
# DISKSUITE SECTION #
#################################################################
if [ $DISKSUITE_PRESENT -ne 0 ]
then
   $METASTAT_CMD | $GREP_CMD $GREP_ARG aint
   if [ $? -eq 0 ]
   then
      OUTPUT_MSG="${OUTPUT_MSG}Disk requires maintenance on `uname -n`\n"
   fi
   $METASTAT_CMD | $GREP_CMD $GREP_ARG "In use"
   if [ $? -eq 0 ]
   then
      OUTPUT_MSG="${OUTPUT_MSG}Disk hot spared on `uname -n`"
   fi
   $METADB_CMD | $GREP_CMD $GREP_ARG "[A-Z]"
   if [ $? -eq 0 ]
   then
      OUTPUT_MSG="${OUTPUT_MSG}Metadb problems on `uname -n`\n"
   fi
fi
#################################################################
# OUTPUT SECTION #
#################################################################
if [[ ! -z $MAIL_OUTPUT ]]
then
   if [[ ! -z "$OUTPUT_MSG" && ! -z $MAIL_RECIP ]]
   then
      print $OUTPUT_MSG | mailx -s "$SUBJECT" $MAIL_RECIP
      OUTPUT_CODE=-1
   fi
   else
   if [[ ! -z "$OUTPUT_MSG" ]] # or else just print on stdout
   then
      print $OUTPUT_MSG
      OUTPUT_CODE=-1
   fi
fi
return $OUTPUT_CODE