#!/bin/sh

# failover
#
# BIG BROTHER - FAILOVER SCRIPT
# Sean MacGuire
# Version 1.9
# Mar 13th, 2002
#
# (c) Copyright Quest Software, Inc.  1997-2002  All rights reserved.
#

#
# failover WATCHES BBNET and BBPAGER 
#
# IF BBNET OR BBPAGER BECOMES UNAVAILABLE, THEN TAKE OVER UNTIL THEY RETURN
#
# To use, just add failover to the BBEXT variable in etc/bbdef.sh
#
# To configure BBPAGER failover:
# define both the primary and failover machines as BBPAGERS in etc/bb-hosts 
# and set bbwarn: FAILOVER in etc/bbwarnsetup.cfg
#
# BBNET failover is automatic when we can't reach the BBNET machine
# BBNET should only be defined ONCE in the bb-hosts file, and should
# not be defined on the failover machine (that's this one if you're
# reading this!).
#
# However, if your BBNET machine is also your primary BBPAGER, then 
# you can have BBNET failover when BBPAGER failsover.  This is useful
# to detect the situation when the BBNET machine is still reachable but
# BBNET isn't working because someone turned BB off...  To enable this
# behaviour, uncomment the following line:

# FAILOVERBOTH=TRUE		# FAILOVER BBNET WITH BBPAGER

BBPROG=failover; export BBPROG
#
TEST="fo"
COLOR="clear"				# ASSUME ALL IS WELL...

#
# BBHOME CAN BE SET MANUALLY WHEN TESTING.
# OTHERWISE IT SHOULD BE SET FROM THE BB ENVIRONMENT
#
# BBHOME=/home/sean/bb; export BBHOME	# FOR TESTING

if test "$BBHOME" = ""
then
	echo "BBHOME is not set... exiting"
	exit 1
fi

if test ! "$BBTMP"                      # GET DEFINITIONS IF NEEDED
then
	 # echo "*** LOADING BBDEF ***"
        . $BBHOME/etc/bbdef.sh          # INCLUDE STANDARD DEFINITIONS
fi

#
# BBPAGER FAILOVER 
#
# IF WE'RE A BBPAGER FAILOVER, THEN YOU'LL NEED 2
# BBPAGERS DEFINED IN etc/bb-hosts (PREFERABLY WITH 
# THE SAME RULES).  THIS MACHINE *MUST* BE DEFINED AS
# A BBPAGER FOR FAILOVER TO WORK.
#
# IF BBPAGER FAILOVER DOESN'T SEEM TO BE WORKING, THEN UNCOMMENT THE
# NEXT LINE ONCE YOU'VE SET THE IP-ADDRESS OF YOUR PRIMARY BBPAGER 
# BBPAGE=123.123.123.123	# ENTER THE IP ADDR OF PRIMARY BBPAGER HERE
#
if test "$BBPAGERS"
then
	# echo "BBPAGERS: $BBPAGERS 	BBPAGE: $BBPAGE"
	# WE'LL SEND A PAGE TO ALL THE BBPAGERS
	# IF ANY FAIL WE'LL GO INTO FAILOVER MODE... THIS DOESN'T MATTER
	# BECAUSE IF THE FAILOVER FAILS IT MAKES NO DIFFERENCE!
	#
	$BB $BBPAGE "dummy ${MACHINE}.${TEST} failover test" > $BBTMP/FO.$$ 2>&1
	if test -s "$BBTMP/FO.$$"		# SOMETHING FAILED...
	then
		$BB $BBPAGE "dummy ${MACHINE}.${TEST} failover test" > $BBTMP/FO.$$ 2>&1
		if test -s $BBTMP/FO.$$		# AND FAILED AGAIN...
		then
			if test ! -f $BBTMP/FAILOVER-BBPAGER
			then
				date > $BBTMP/FAILOVER-BBPAGER
			fi
		fi
	fi
	
	if test ! -s "$BBTMP/FO.$$"			# PAGING IS ALIVE
	then
		if test -f "$BBTMP/FAILOVER-BBPAGER"	# FAILBACK
		then
			$RM -f $BBTMP/FAILOVER-BBPAGER
		fi

		if test "$FAILOVERBOTH" = "TRUE"
		then
			BBNETFO=""			# RESTORE BBNET FAILOVER
		fi

		BBPLINE="&clear BBPAGER on standby"
	else
		if test "$FAILOVERBOTH" = "TRUE"
		then
			BBNETFO="TRUE"			# FORCE BBNET FAILOVER
		fi

		COLOR="yellow"
		BBPLINE="&yellow BBPAGER failover since `cat $BBTMP/FAILOVER-BBPAGER`"
	fi

	$RM -f $BBTMP/FO.$$
fi

#
# IS THE BBNET MACHINE ALIVE?
# SEE IF WE CAN PING IT... 
#
BBNET=`$GREP BBNET $BBHOSTS | $GREP "^[0-9]" | $GREP -v "^\#"`
if test "$BBNET"
then
	set $BBNET
	BBNET="$2"
	# echo "BBNET is $BBNET"
	# BBNET="bobo.bb4.com" 	# TO TEST BROKEN BBNETS
fi

LINE=`$BBHOME/bin/bb-ping.sh $BBNET FALSE`

if test "$?" != "0" || test "$BBNETFO" = "TRUE"
then
	COLOR="yellow"
	#
	# NOW WE HAVE TO FAILOVER... ONCE.
	# PHILOSOPHICAL QUESTION AS TO WHAT COLOR THE DOT SHOULD BE
	# WHEN WE'RE IN FAILOVER MODE...  MAYBE YELLOW (warning...)
	# OTHERWISE FAILOVER IS ON STANDBY (clear)
	#
	# BACKUP THE BBPID FILE
	if test ! -f $BBTMP/FAILOVER-BBNET
	then
		BBSLEEP=300; export BBSLEEP
		cp $BBPID $BBTMP/FAILOVER-BBNET
       		# echo "*** CALLING BB-NETWORK ***"
        	{ nohup $BBHOME/bin/bbrun $BBHOME/bin/bb-network.sh ;} >> $BBOUT 2>&1 &
	fi
	BBNETLINE="&yellow BBNET failover running"
else
	#
	# IF WE'RE BETTER, THEN WE CAN FAILBACK (IS THAT A WORD?)
	# 
	if test -f $BBTMP/FAILOVER-BBNET
	then
		# KILL THE LAST PID 
		kill -9 `tail -1 $BBPID`		# TOAST bb-network.sh
		cp $BBTMP/FAILOVER-BBNET $BBPID
		rm -f $BBTMP/FAILOVER-BBNET
	fi
	BBNETLINE="&clear BBNET on standby"
fi

if test "$COLOR" = "yellow"
then
	LINE="Failover is running"
else
	LINE="Failover on standby"
fi

LINE="$LINE
$BBPLINE
$BBNETLINE"

$BB $BBDISP "status ${MACHINE}.${TEST} $COLOR `date` $LINE"
