#!/bin/sh
#
# @(#)$Id: fbs,v 1.64 2003/08/22 17:25:09 ivm Exp $
#
# $Log: fbs,v $
# Revision 1.64  2003/08/22 17:25:09  ivm
# Tested min. nice for proctype
#
# Revision 1.63  2002/09/18 17:35:58  ivm
# Removed debug output
# Added fbswww scripts
# Added "fbs fbswww" subcommand
#
# Revision 1.62  2001/12/18 20:52:22  ivm
# Added args to "fbs submit help"
#
# Revision 1.61  2001/11/05 19:22:03  ivm
# Fixed some bugs
# Made inetractive spawn7er independent of UPS
#
# Revision 1.60  2001/09/10 20:44:32  ivm
# Removed debug print-outs
#
# Revision 1.59  2001/05/25 21:39:37  ivm
# Implemented hold node and disconnect
#
# Revision 1.58  2001/04/11 16:38:28  ivm
# Added ptnodes.py
#
# Revision 1.57  2001/02/06 18:25:25  ivm
# Fixed yet another memory mishadnling in farm_history, simplified syntax
# Added canceled() dependency type
# Added job ranges in kill.py
# Show processes for just ended sections
# Fixed decoding of status/signal/core in Section
#
# Revision 1.56  2000/12/22 17:21:48  ivm
# Removed postmortem launcher output print-out
#
# Revision 1.55  2000/12/19 21:00:25  ivm
# Fixed fbs script
# Create shm file in createEnv() in Launcher
# Look for launcher.pid in /fbsng/tmp in kill_lch.sh
#
# Revision 1.53  2000/11/30 20:23:19  ivm
# Fixed bugs
# Made Scheduler more conservative about unknown queues/ptypes
# Use /tmp/launcher.pid for launcher inter-locking
#
# Revision 1.52  2000/11/15 21:03:15  tlevshin
# added -m option for xsysmon
#
# Revision 1.51  2000/11/14 19:50:27  ivm
# Added "fbs hold/release queue all" to help
#
# Revision 1.50  2000/11/08 20:27:02  ivm
# Implemented FBSClient.getServerOptions()
# Fixed bugs
#
# Revision 1.49  2000/10/06 18:11:27  ivm
# Fixed syntax error in JobFinder
# Removed splitcfg from fbs
#
# Revision 1.47  2000/10/04 17:23:35  ivm
# Added and fixed interactive stuff
#
# Revision 1.46  2000/10/02 14:40:03  tlevshin
# change help for xsysmon
#
# Revision 1.45  2000/09/29 20:46:06  fromm
# Added lock/unlock for queues.
#
# Revision 1.43  2000/09/20 14:02:46  ivm
# Fixed minor bugs
#
# Revision 1.41  2000/08/29 17:06:45  ivm
# Fixed some bugs
#
# Revision 1.40  2000/08/25 16:34:58  ivm
# Added splitcfg.py to Makefile
#
# Revision 1.38  2000/08/21 20:34:34  ivm
# Renamed config.py -> farm_config.py
#
# Revision 1.36  2000/08/18 21:15:43  ivm
# Implemented dynamic re-configuration
#
# Revision 1.35  2000/07/06 20:35:32  ivm
# Added -v to status
#
# Revision 1.34  2000/07/05 20:11:55  ivm
# Fixed usage for hold/release
#
# Revision 1.31  2000/07/03 20:59:25  ivm
# Added usage for "ls" and "lj"
#
# Revision 1.27  2000/06/26 19:41:32  ivm
# Check if history archive file exists
#
# Revision 1.26  2000/06/26 15:40:18  ivm
# Launcher gets cfg file from environment
#
# Revision 1.25  2000/06/23 19:40:13  tlevshin
# implemented archiving
#
# Revision 1.24  2000/06/22 18:58:18  ivm
# Report syntax error in intelligent way in proc_types and resources.
# Pass strings as they are to proc_types and resources
#
# Revision 1.23  2000/06/15 21:37:35  ivm
# Added slog
#
# Revision 1.22  2000/06/12 18:41:20  ivm
# Added xsysmon
#
# Revision 1.21  2000/06/09 15:43:27  ivm
# Added cprio
#
# Revision 1.20  2000/06/08 21:53:47  ivm
# Fixed getcfg
#
# Revision 1.19  2000/06/02 19:44:02  ivm
# Added jobs and sections (lj and ls) to fbs script. Still "help" to be done.
# Changed version to v0_6
#
# Revision 1.18  2000/05/31 16:34:43  ivm
# Changed farms to fbs in history help
#
# Revision 1.17  2000/05/30 18:49:56  ivm
# Fixed syntax error
#
# Revision 1.16  2000/05/24 14:12:08  ivm
# Added getcfg, history to fbs script
#
# Revision 1.15  2000/05/23 21:57:27  ivm
# Run chkcfg before bmgr
#
# Revision 1.14  2000/05/23 21:51:33  ivm
# Added chkcfg
#
# Revision 1.13  2000/05/23 16:51:51  ivm
# Fixed hold/release
#
# Revision 1.11  2000/05/16 17:04:43  ivm
# Fixed "msg"
#
# Revision 1.10  2000/05/11 17:25:04  ivm
# Added "msg"
#
# Revision 1.9  2000/05/08 20:56:30  ivm
# Replaced tabs with spaces
#
# Revision 1.7  2000/05/08 19:05:50  ivm
# Fixed launcher
#
# Revision 1.4  2000/04/26 16:33:18  ivm
# *** empty log message ***
#
# Revision 1.3  2000/04/25 16:10:24  ivm
# Added resources.py
#
# Revision 1.1  2000/04/11 15:36:44  ivm
# Added fbs
#
#
#******************************************************************
# FBS UI wrapper script                                           *
#******************************************************************

#++NON-FUE
# add python to the path
# add FBS to python path
# define FBS env. variables
#--NON-FUE

lch_tmp_dir="/fbsng/tmp"
lch_log_dir="/fbsng/log"
bmgr_log_dir="$FBSNG_ROOT/bmgr"

if [ "${1:-}" = "-x" ] ; then set -xv; shift; fi

if [ ! -z "${FBSNG_PYTHON}" ] ; then
	alias python='${FBSNG_PYTHON}'
fi


usage="
----------------------- FBS User Interface Summary ----------------------------
Job manipulation
    fbs submit            -- submit a job
    fbs wait              -- wait for a job/section to finish
    fbs lj                -- list jobs
    fbs ls                -- list sections
    fbs status            -- print job/section status
    fbs slog              -- print section log output
    fbs history           -- print job/section history
    fbs kill              -- kill a job/section
    fbs cprio             -- increase/decrease section priority
Farm status monitoring/modifying, other FBS information
    fbs monitor           -- monitor jobs/FBS
    fbs nodes             -- show node status
    fbs xsysmon           -- monitor resource utilization
    fbs sysmon            -- print resource utilization
    fbs hold              -- hold node, queue, job or section
    fbs release           -- release node, queue, job or section
    fbs lock              -- lock queue
    fbs unlock            -- unlock queue
    fbs queues            -- show queues
    fbs ptypes            -- show process types
    fbs resources         -- show resource utilization statistics
    fbs msg               -- send status message from batch process
    fbs allow/disallow    -- dis(allow) processes of certain type on nodes
-------------------------------------------------------------------------------
See also:
    fbs <command> help    -- for more details on specific command usage
    fbs help_admin        -- for FBS adminitrator commands
"
extra_usage="

****************************************************************************
*** THE FOLLOWING COMMANDS SHOULD BE USED BY FBS ADMINISTRATORS ONLY !!! ***
*** READ FBSNG ADMINISTRATION GUIDE BEFORE USING ANY OF THESE COMMANDS   ***
****************************************************************************

Starting FBS daemons
    fbs launcher
    fbs bmgr 
    fbs logd
    fbs fbswww

Configuration
    fbs config                          -- interactive mode
    fbs config <command> [<args> ...]   -- command line mode
    fbs chkcfg
    fbs getcfg <set> (<id>|.host) <field>
"

cmd="help"

if [ "_$1" != "_" ] ; then
    cmd="$1"
    shift
fi
stat=0

if [ "_$cmd" = "_help" -o "_$cmd" = "_--help" ]; then
	echo "$usage"
	exit 0
fi

if [ "_$cmd" != "_config" -a \( "_$1" = "_help" -o "_$1" = "_--help" \) ]; then
	#
	# Print subcommand usage
	#
	case $cmd in
        allow)
            echo 'fbs allow <proc_type> (-|*|<node> ...)'
            ;;
        disallow)
            echo 'fbs disallow <proc_type> (-|*|<node> ...)'
            ;;

		exec)
			echo "fbs exec -q <queue> [-I] [-v] [-n <numproc>]"
		 	echo "  [-p <proctype>] [-w <seconds or -1>] [-h <hold-date-time>]"
			echo "  [-r <resources>] [-i <priority increment>] [-s <section-name>]"
			echo "  <command> [<argument> ...]"
			;;
        slog)       echo "fbs slog <sectionid>"
            ;;
		submit) 	
			echo "fbs submit [-1] [-w [-t <timeout>]] [-o <override_string>] ... [<jdf_file>]"
			echo "fbs submit [-1] [-w [-t <timeout>]] <jdf_file> [<arg1> ...]"
			;;
		kill)		echo "fbs kill [-i] ([-s <sect-name> <jobid> ... |<sectionid> ...)"
			;;
		status) 	echo "fbs status [-u (<user>|all)] [-v] [(<jobid>|<sectionid>) ...]"
			;;
		hosts)		echo "fbs hosts [-d] [-l] [-t '<class> ...'] [<node> ...]" 
			;;
		nodes)		echo "fbs nodes [-d] [-l] [-t '<class> ...'] [<node> ...]" 
			;;
		wait)		echo "fbs wait  [-t <time_out>] ([-s <section>] <jobid>|<sectionid>)"
			;;
		monitor)	echo "fbs monitor"
			;;
		xsysmon)	echo "farms xsysmon [(<host>...|<host_prefix> <start> <end>|-j (<jobid>|<sectionid>))]"
			;;
		#sysmon) 	echo "fbs sysmon [-a|-cpsdxlTiIOeDCg]"
        #            echo "             [-r <interval> [-n <times>]] <host> ..."
		#	;;
		queues) 	echo "fbs queues [-l] [<queue_name> ...]" 
			;;
		ptypes) 	echo "fbs ptypes [-lruq] [<process_type> ...]" 
			;;
		resources)	
			echo "fbs resources [-l][-g]"
			echo "              -p <proc_type> [<resource> ...]"
			echo "              -n <node> [<resource> ...]"
			;;
		hold)
				echo "fbs hold node [-r '<reason>'] [-d] <node> ..."
				echo "fbs hold queue (all|<queue> ...)"
				echo "fbs hold (job|section) [-t <time>] (<jobid>|<sectionid>) ..."
			;;
		lock)
				echo "fbs lock queue <queue> ..."
			;;
		history)	echo "fbs history [-b <begin_time>] [-e <end_time>]"
					echo "   [-u ('<user> ...'|all)] [-s '<section> ...']"
					echo "   [-h '<host> ...'] [-j ('<jobid>|<section-id>) ...']"
					echo "   [-t '<process_type> ...']"
			;;
		release)
				echo "fbs release node <node> ..."
				echo "fbs release queue (all|<queue> ...)"
				echo "fbs release (job|section) (<jobid>|<sectionid>) ..."
			;;
		unlock)
				echo "fbs unlock queue <queue> ..."
			;;
		msg)	echo "fbs msg '<message>'"
			;;
		getcfg) 
			echo "fbs getcfg <set-name> <set-id> <field-name>"
			echo "fbs getcfg <set-name> .node <field-name>"
			;;
		cprio)
			echo "fbs cprio [+]<n> <sectionid> ..."
			echo "fbs cprio -<n> <sectionid> ..."
			;;
		ls)
			echo "fbs ls [-u (<user>|all)]"
			echo "       [-q <queue>] [-s <states>]"
			echo "       [(<jobid>|<sectionid>) ...]"
			;;
		lj) echo "fbs lj [-u (<user>|all)] [<jobid> ...]"
			;;
		*)	echo "Help for \"$cmd\" is unavailable."
			echo ""
			echo "$usage"
			;;
	esac	
else
	#
	# Execute subcommand
	#
	case $cmd in
        allow)
            python $FBSNG_DIR/bin/ptnodes.py allow "$@"
            exit $?
            ;;
        disallow)
            python $FBSNG_DIR/bin/ptnodes.py disallow "$@"
            exit $?
            ;;
		exec)	python $FBSNG_DIR/bin/exec.py "$@"
			;;
		help_admin)
			echo "$extra_usage"
		;;
		config) python $FBSNG_DIR/bin/farm_config.py "$@"
			exit $?
			;;
        slog)   python $FBSNG_DIR/bin/sect_output.py $@
				exit $?
            ;;
    	xsysmon) python $FBSNG_DIR/bin/GuiRstat.py -m Rstat $@ 
				exit $?
		;;
    	#sysmon) python $FBSNG_DIR/bin/farm_sysmon.py $@ 
		#;;
		cprio)	python $FBSNG_DIR/bin/cprio.py -i $@
				exit $?
			;;
		ls) python $FBSNG_DIR/bin/sections.py "$@"
			exit $?
			;;
		lj) python $FBSNG_DIR/bin/jobs.py $@
			exit $?
			;;
		getcfg) python $FBSNG_DIR/bin/getconfig.py $FBS_CONFIG $@
				exit $?
			;;

    	submit) python $FBSNG_DIR/bin/submit.py "$@"
		  exit $?
		;;
    	kill) python $FBSNG_DIR/bin/kill.py  $@
		  exit $?
        	;;
    	status) python $FBSNG_DIR/bin/status.py $@ 
		  exit $?
		;;
    	nodes|hosts) python $FBSNG_DIR/bin/hosts.py $@ 
			  exit $?
		;;
		resources)	python $FBSNG_DIR/bin/resources.py "$@"
			  exit $?
		;;
 		ptypes)	python $FBSNG_DIR/bin/proc_types.py "$@"
				exit $?
		;;
   		wait) python $FBSNG_DIR/bin/wait.py $@ 
		  exit $?
		;;
	   	monitor) python $FBSNG_DIR/bin/GuiFarmsMenu.py $@ 
		;;
    	launcher) 	
		# make sure lch_tmp and lch_log exist
		if [ ! -d $lch_tmp_dir ] ;then
			mkdir -p $lch_tmp_dir
		fi
		chmod go+rx $lch_tmp_dir

		if [ ! -d $lch_log_dir ] ;then
			mkdir -p $lch_log_dir
		fi
		chmod go+rx $lch_log_dir
		
		# open new output and error files
		rm -f $lch_log_dir/launcher.out.9 2> /dev/null		
		mv -f $lch_log_dir/launcher.out.8 $lch_log_dir/launcher.out.9 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.7 $lch_log_dir/launcher.out.8 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.6 $lch_log_dir/launcher.out.7 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.5 $lch_log_dir/launcher.out.6 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.4 $lch_log_dir/launcher.out.5 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.3 $lch_log_dir/launcher.out.4 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.2 $lch_log_dir/launcher.out.3 2> /dev/null
    		mv -f $lch_log_dir/launcher.out.1 $lch_log_dir/launcher.out.2 2> /dev/null
    		mv -f $lch_log_dir/launcher.out   $lch_log_dir/launcher.out.1 2> /dev/null

		cd $FBSNG_DIR/bin
		exec python launcher.py $@ >$lch_log_dir/launcher.out 2>&1
		sts=$?
		#tail -100 $lch_log_dir/launcher.out
		exit $sts
		;;
    	bmgr) 
			# make sure lch_tmp and lch_log exist
			if [ ! -d $bmgr_log_dir ] ;then
				mkdir -p $bmgr_log_dir
			fi
			chmod go+rx $bmgr_log_dir
			rm -f $bmgr_log_dir/bmgr.out.9 2> /dev/null		
			mv -f $bmgr_log_dir/bmgr.out.8 $bmgr_log_dir/bmgr.out.9 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.7 $bmgr_log_dir/bmgr.out.8 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.6 $bmgr_log_dir/bmgr.out.7 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.5 $bmgr_log_dir/bmgr.out.6 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.4 $bmgr_log_dir/bmgr.out.5 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.3 $bmgr_log_dir/bmgr.out.4 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.2 $bmgr_log_dir/bmgr.out.3 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out.1 $bmgr_log_dir/bmgr.out.2 2> /dev/null
			mv -f $bmgr_log_dir/bmgr.out   $bmgr_log_dir/bmgr.out.1 2> /dev/null

			if python $FBSNG_DIR/bin/chkcfg.py $FBS_CONFIG $FBS_FARM_CONFIG ;then
				cd $FBSNG_DIR/bin
				exec python bmgr.py >$bmgr_log_dir/bmgr.out 2>&1
			else
				echo "FBS configuration verification failed. BMGR did not start."
			fi
			exit $?
		;;
    	logd) python $FBSNG_DIR/bin/logd.py -c $FBS_CONFIG
				exit $?
        	;;

        fbswww) python $FBSNG_DIR/bin/fbswww.py >/dev/null </dev/null 2>/dev/null
                exit $?
                ;;

    	history) 
			hist_file=`fbs getcfg history - hist_file`
			hist_dir=`fbs getcfg history - hist_dir`
			archive_file=`fbs getcfg history - archive_file`
			if [ "${archive_file}" = "None" ] ;then
				archive_file=""
			else
				archive_file="${hist_dir}/${archive_file}"
			fi
			if [ -z "${archive_file}" -o ! -r "${archive_file}" ] ;then
				archive_file="/dev/null"
			fi
			$FBSNG_DIR/bin/farm_history "$@" ${archive_file} ${hist_dir}/${hist_file}
			exit $?
		;;
    	hold)
			what="$1"
			shift
			case $what in
				node)	python $FBSNG_DIR/bin/nodeadmin.py -H "$@"
						exit $?
					;;
				queue)	python $FBSNG_DIR/bin/queueadmin.py -h $@		
						exit $?
					;;
				job|section)	
						python $FBSNG_DIR/bin/jobadmin.py -H $@		
						exit $?
					;;
				*)		echo "Unrecognized keyword $what"
						echo "Type \"fbs hold help\" for usage"
						exit 1
					;;
			esac
			;;
		release) 
			what="$1"
			shift
			case $what in
				node)	python $FBSNG_DIR/bin/nodeadmin.py -R $@
						exit $?
					;;
				queue)	python $FBSNG_DIR/bin/queueadmin.py -r $@		
						exit $?
					;;
				job|section)	python $FBSNG_DIR/bin/jobadmin.py -R $@		
						exit $?
					;;
				*)		echo "Unrecognized keyword $what"
						echo "Type \"fbs release help\" for usage"
						exit 1
			esac
			;;

                lock)
			what="$1"
			shift
			case $what in
				queue)	python $FBSNG_DIR/bin/queueadmin.py -l $@		
						exit $?
					;;
				*)		echo "Unrecognized keyword $what"
						echo "Type \"fbs lock help\" for usage"
						exit 1
			esac
			;;
                
		unlock)
			what="$1"
			shift
			case $what in
				queue)	python $FBSNG_DIR/bin/queueadmin.py -u $@		
						exit $?
					;;
				*)		echo "Unrecognized keyword $what"
						echo "Type \"fbs unlock help\" for usage"
						exit 1
			esac
 			;;
                           
		queues) python $FBSNG_DIR/bin/queues.py $@
				exit $?
			;;
		msg)
			if [ -z "${FBS_SHM_KEY}" ]; then
				echo "FBS_SHM_KEY variable not defined."
				echo "fbs msg command can be executed only from batch environment."
				exit 1
			fi
			r2shm p -f $FBS_SHM_KEY "$@"
			;;
		chkcfg) python $FBSNG_DIR/bin/chkcfg.py $FBS_CONFIG $FBS_FARM_CONFIG $@
				exit $?
			;;
		#splitcfg) python $FBSNG_DIR/bin/splitcfg.py
		#	exit $?
		#	;;
    	*) echo "$usage"
    	   exit 1
    	   ;;
	esac
fi				# end if arg == "help"
exit 0
