#!/bin/sh

# Globus Job Manager FBSNG interface script for submitting a job status
# Author:              Anzar Afaq & Igog Mandrichenko (FNAL)   
# Last Modified:       08/09/2002
#
# This script builds a shell job script which is supplied as input
# to the LSF bsub command. The script is built based on information
# obtained from a file passed as the script's argument. This file
# contains a list of environment variables which are set by way
# of "sourcing" the file from this script. The evironment variables
# set as a result of this action are then used to characterize the
# user's job request. Once the job script has been submitted the
# LSF job id is appended to the file passed as an argument to this
# script to be used by other scripts at a later time.

# The temporary job script is created in a location defined by the
# TMP_DIR and LSF_JOB_SCRIPT name variables. This file is removed at
# the end of this script. 

	
. ${GLOBUS_LOCATION}/libexec/globus-script-initializer
 
globus_source ${libexecdir}/globus-gram-protocol-constants.sh
globus_source ${libexecdir}/globus-sh-tools.sh
globus_source ${libexecdir}/globus-gram-job-manager-tools.sh

awk=${GLOBUS_SH_AWK-awk}
rm=${GLOBUS_SH_RM-rm}
SH=/bin/sh

#-------------------------------------------------------
# FBSNG Job Manager settings
#-------------------------------------------------------
# Important: Edit these definitions according to your local setup
 
# $FBSNG_SETUP_RUN_QUEUE - default queue to use for the job

FBSNG_SETUP_RUN_QUEUE=GRAM

# $FBSNG_SETUP_CLEANUP_QUEUE - default queue to use for the job

FBSNG_SETUP_CLEANUP_QUEUE=GlobusClean

# $FBSNG_SETUP_IO_DIR - NFS-shared directory for temporary storage of
#                       scripts and stderr/stdout files
#						This directory should be writeable by any user

FBSNG_SETUP_IO_DIR=/data/fbsng_scratch

#---------------------------------------------
# Set up FBSNG environment
#---------------------------------------------

. /local/ups/etc/setups.sh >/tmp/setups.out 2>/tmp/setups.err
setup fbsng
setup kerberos

MY_ID="$$.`date +%m-%d-%y-%H:%M:%S`"		# add time

FBSNG_IO_DIR=${FBSNG_SETUP_IO_DIR}
FBSNG_JOB_DIR=${FBSNG_IO_DIR}/${MY_ID}.job
FBSNG_JOB_SCRIPT=${FBSNG_JOB_DIR}/job_script.sh
FBSNG_JOB_OUT=${FBSNG_JOB_DIR}/job_out
FBSNG_JOB_ERR=${FBSNG_JOB_DIR}/job_err
FBSNG_STDOUT=${FBSNG_JOB_DIR}/proc.out
FBSNG_STDERR=${FBSNG_JOB_DIR}/proc.err
FBSNG_JDF=${local_tmpdir}/Globus_${MY_ID}.jdf
FBSNG_NUMPROC=$grami_count
FBSNG_CLEAN_SCRIPT=${FBSNG_JOB_DIR}/globus_clean.sh
arg_file=$1

#        Check for the argument file. If it does not exist
#        then return with an error immediately

if [ ! -f $arg_file ] ; then
   echo GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_BAD_SCRIPT_ARG_FILE
   exit 1
fi
 
#        Source the argument file to set environment variables
#        defining all the job arguments

. $arg_file

#        If a logfile name has been defined then activate debug mode

#if [ $grami_logfile = "/dev/null" ] ; then
#    DEBUG_ECHO=:
#else
DEBUG_ECHO=echo
#fi

#echo Debug echo is $DEBUG_ECHO
#echo grami_logfile: $grami_logfile


grami_logfile=/tmp/anzar.tmp

$DEBUG_ECHO "JM_SCRIPT: in fbsng_submit">> $grami_logfile

#progcontent=`cat $grami_program`

$DEBUG_ECHO "JM_SCRIPT: Analysing some input parameters">> $grami_logfile    
$DEBUG_ECHO "JM_SCRIPT: program file contents:-">> $grami_logfile   
$DEBUG_ECHO "JM_SCRIPT: $progcontent" >>  $grami_logfile 
$DEBUG_ECHO "JM_SCRIPT: End of Analysing">> $grami_logfile


$DEBUG_ECHO ""                                            >> $grami_logfile
$DEBUG_ECHO ============================================  >> $grami_logfile
$DEBUG_ECHO "JM_SCRIPT: ====argument file contents===="   >> $grami_logfile
if [ "$DEBUG_ECHO" = "echo" ] ; then
   cat $arg_file                                          >> $grami_logfile
fi
$DEBUG_ECHO "JM_SCRIPT: ====argument file contents===="   >> $grami_logfile
$DEBUG_ECHO ""                                            >> $grami_logfile

#        The following several lines of code can be used to perform 2
#        additional error checks prior to job submission. The first check is
#        for the existance of the directory which the user requested
#        be the working directory. If it does not exist the script
#        returns an error and the job is not submitted. The second check
#        is for existance of the file requested by the user to be used for
#        stdin. If the file does not exist the scripts returns an error and
#        the job is not submitted.

#        These checks are only valid if performed on the file system to be used
#        by the host on which the job will run. But, this file system may not
#        be shared with host from which the job is submitted. Therefore, the
#        check does not make sense. If however the host from which the job 
#        will be submitted (i.e. the host running the globus gatekeeper)
#        shares file systems with all the hosts which may potentially
#        run the job these checks can be used. In order to have the job
#        manager perform these checks the following 2 sections of code
#        should *not* be commented out.


#        Check for existance of directory

#$DEBUG_ECHO "JM_SCRIPT: testing for existance of directory $grami_directory" >> $grami_logfile
#if [ -d $grami_directory ]; then
#   $DEBUG_ECHO "JM_SCRIPT: directory $grami_directory found" >> $grami_logfile
#    cd $grami_directory
#else
#   $DEBUG_ECHO "JM_SCRIPT: directory $grami_directory DOES NOT exist; exiting with exit code 1" >> $grami_logfile
#   echo GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_BAD_DIRECTORY
#   exit 1
#fi
#
#         Check for existance of stdin file if not /dev/null
#
#if [ $grami_stdin != "/dev/null" ]; then
#    $DEBUG_ECHO "JM_SCRIPT: testing for existance of stdin file $grami_stdin" >> $grami_logfile
#    if [ -r $grami_stdin ]; then
#        $DEBUG_ECHO "JM_SCRIPT: stdin file $grami_stdin found and is readable" >> $grami_logfile
#    else
#        $DEBUG_ECHO "JM_SCRIPT: stdin file $grami_stdin DOES NOT exist or is not readable; exiting with exit code 1" >> $grami_logfile
#        echo GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_STDIN_NOT_FOUND
#        exit 1
#    fi
#fi


#	Check for non supported parameters here. That is, if any of the RSL
#	parameters which LSF can not support have been requested return an error.
#	Currently all RSL attributes are supported by LSF

# if (unsupported parameters found)
#     $DEBUG_ECHO "JM_SCRIPT: unsupported parameters found. Exiting with error." >> $grami_logfile
# else
#     $DEBUG_ECHO "JM_SCRIPT: No unsupported parameters found" >> $grami_logfile
# fi
#

$DEBUG_ECHO "JM_SCRIPT: testing for unsupported parameters" >> $grami_logfile
$DEBUG_ECHO "JM_SCRIPT: No unsupported parameters found" >> $grami_logfile

        # Verify existance of queue if queue parameter is not NULL

$DEBUG_ECHO "JM_SCRIPT: testing for queue attribute specification" >> $grami_logfile


# default queue
if [ -z "$grami_queue" ] ;then
	$DEBUG_ECHO "JM_SCRIPT: Using default queue $FBSNG_SETUP_RUN_QUEUE"
	grami_queue=$FBSNG_SETUP_RUN_QUEUE
fi

$DEBUG_ECHO "JM_SCRIPT: comments by anzar --- $grami_queue " >> $grami_logfile
$DEBUG_ECHO "JM_SCRIPT: comments by anzar ---Running command: $fbs_queues " >> $grami_logfile 

echo "LD_LIBRARY_PATH:= $LD_LIBRARY_PATH" >> $grami_logfile 



if [ -z "${grami_queue}" ]; then
    $DEBUG_ECHO "JM_SCRIPT: no queue attribute specified" >> $grami_logfile
    echo GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_INVALID_QUEUE
    exit 1
fi

FBSNG_NUMPROC=${grami_count}
if [ -z "${grami_count}" ] ;then
	FBSNG_NUMPROC="1"
fi

# 4 jobtypes exist               FBSNG result
# -----------------              ------------------
# jobtype 0 = mpi         -----> ERROR
# jobtype 1 = single      -----> submit job with NPROC = 1
# jobtype 2 = multiple    -----> submit job
# jobtype 3 = condor      -----> ERROR

$DEBUG_ECHO "JM_SCRIPT: testing jobtype" >> $grami_logfile
if [ $grami_job_type = "0" ] ; then
   $DEBUG_ECHO "JM_SCRIPT: ERROR: jobtype parameter not supported"
               >> $grami_logfile
   echo "GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_JOBTYPE_NOT_SUPPORTED"
   exit 1
elif [ $grami_job_type = "1" ] ; then
    fbsng_jobtype="single"
	FBSNG_NUMPROC="1"
elif [ $grami_job_type = "2" ] ; then
    fbsng_jobtype="multiple"
elif [ $grami_job_type = "3" ] ; then
   $DEBUG_ECHO "JM_SCRIPT: ERROR: jobtype parameter not supported"
               >> $grami_logfile
   echo "GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_JOBTYPE_NOT_SUPPORTED"
   exit 1
else
   $DEBUG_ECHO "JM_SCRIPT: invalid jobtype parameter" >> $grami_logfile
   echo GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_CLIENT_ERROR_INVALID_JOBTYPE
   exit 1
fi

#         Determining cpu time limit

$DEBUG_ECHO "JM_SCRIPT: ignoring cpu time limit" >> $grami_logfile

#       Determining memory limit
#	min memory is *NOT* supported
#       Globus default RSL attribute memory units are in Mbytes
#       LSF default memory attribute units are in Kbytes...
#       so a conversion from Mbytes to Kbytes must be made

Kb=1024

$DEBUG_ECHO "JM_SCRIPT: ignoring memory limit" >> $grami_logfile
#if [ $grami_max_memory -eq 0 ] ; then
#    max_memory=0
#    $DEBUG_ECHO "JM_SCRIPT: no maximum memory requested specified" >> $grami_logfile
#else
#    max_memory=`expr "$grami_max_memory" \* "$Kb"`
#    $DEBUG_ECHO "JM_SCRIPT: requested $grami_max_memory Mb for maximum memory" >> $grami_logfile
#    $DEBUG_ECHO "JM_SCRIPT: converting Mb to LSF default unit of Kb" >> $grami_logfile
#    $DEBUG_ECHO "JM_SCRIPT: using $max_memory Kb for maximum memory" >> $grami_logfile
#fi
#
#$DEBUG_ECHO "JM_SCRIPT: testing for minimum memory limit" >> $grami_logfile
#if [ $grami_min_memory -eq 0 ] ; then
#    $DEBUG_ECHO "JM_SCRIPT: no minimum memory requested specified" >> $grami_logfile
#else
#    $DEBUG_ECHO "JM_SCRIPT: requested $grami_min_memory Mb for minimum memory" >> $grami_logfile
#    $DEBUG_ECHO "JM_SCRIPT: minimum memory request is not supported" >> $grami_logfile
#    $DEBUG_ECHO "JM_SCRIPT: minimum memory request is being ignored" >> $grami_logfile
#fi

#-----------------------------------------------------
#	Create job directory
#-----------------------------------------------------
mkdir -p ${FBSNG_JOB_DIR} 2>> $grami_logfile

#-----------------------------------------------------
#         Create JDF
#-----------------------------------------------------

$DEBUG_ECHO "JM_SCRIPT: starting to build job script and JDF" >> $grami_logfile

cat > $FBSNG_JDF << _EOF_
#
# FBSNG JDF built by Globus Job Manager
#
SECTION GlobusMain
  QUEUE = $grami_queue
  NUMPROC = $FBSNG_NUMPROC
  EXEC = $FBSNG_JOB_SCRIPT
  STDOUT = ${FBSNG_STDOUT}.%n
  STDERR = ${FBSNG_STDERR}.%n
#  STDOUT=$grami_stdout
#  STDERR=$grami_stderr
  MAILTO = anzar\@fnal.gov
_EOF_

if [ ! -z "${grami_project}" ] ; then
    echo "  PROC_TYPE = $grami_project" >> $FBSNG_JDF
fi

cat >> $FBSNG_JDF << _EOF_
# Clean-up section name must begin with _
# so that it is not killed with "fbs kill <jobid>"

SECTION _CleanUp		
  QUEUE = ${FBSNG_SETUP_CLEANUP_QUEUE}
  EXEC = $FBSNG_CLEAN_SCRIPT
  STDERR = ${FBSNG_JOB_DIR}/cleanup.err
  STDOUT = ${FBSNG_JOB_DIR}/cleanup.out
  #MAILTO = anzar\@fnal.gov
  DEPEND = ended(GlobusMain) || canceled(GlobusMain)
_EOF_


#-----------------------------------------------------
#	Create Job Script
#-----------------------------------------------------

cat > $FBSNG_JOB_SCRIPT << _EOF_
#!${SH} -f
#

_EOF_





# Below is a while loop to reformat the environment variable 
# string to the format needed for the LSF job script. The environment
# variables for the LSF job will be set during the execution
# of the job script. The loop obtains environment variable information
# from the variable $grami_env. The information is then written
# to the LSF job script one line per variable in the form:

# env_variable=env_value; export env_variable

# where...

# env_variable is the name of the environment variable
# env_value is the value of the environment variable

$DEBUG_ECHO "JM_SCRIPT: checking environment" >> $grami_logfile
#
#loop through all the environment variables.  Variables and values are seperate
#arguments.  While assembling var/value pairs add the specific syntax
#required for this scheduling system.
#
new_grami_env=""
 
if [ ! -z "${grami_env}" ] ; then
   eval set -- ${grami_env}
   x=0
   while [ "$#" -ne 0 ]; do
      if [ $x = 0 ] ; then
         save_variable=$1
         x=1
      else
         x=0
         echo "${save_variable}=$1" >> $FBSNG_JOB_SCRIPT
         echo "export ${save_variable}" >> $FBSNG_JOB_SCRIPT
      fi
 
      shift
   done
fi

new_grami_args=""
if [ ! -z "${grami_args}" ] ; then
   eval set -- ${grami_args}
   new_grami_args="$*"
fi

# Determine directory to be used as working directory 

echo ""  >> $FBSNG_JOB_SCRIPT
echo "# Changing to directory as requested by user" >> $FBSNG_JOB_SCRIPT
echo "cd $grami_directory" >> $FBSNG_JOB_SCRIPT
 
echo ""  >> $FBSNG_JOB_SCRIPT
echo "# Executing job as requested by user" >> $FBSNG_JOB_SCRIPT
echo "$grami_program $new_grami_args <$grami_stdin" >> $FBSNG_JOB_SCRIPT

chmod +x $FBSNG_JOB_SCRIPT

$DEBUG_ECHO "JM_SCRIPT: FBSNG job script successfully built" >> $grami_logfile


wrm=`which rm`
$DEBUG_ECHO "JM_SCRIPT: which rm:=$wrm and set rm:=$rm" >> $grami_logfile 

#-----------------------------------------------------
#	Create Clean-up Script
#-----------------------------------------------------
cat > $FBSNG_CLEAN_SCRIPT << _EOF_
#!${SH} -f
#

PATH=/bin:/usr/bin
export PATH

_EOF_


lpid=0
while [ "$lpid" -lt "$FBSNG_NUMPROC" ] ;do
	lpid=`expr $lpid + 1`
	#echo "echo \"-------- stdout for process #$lpid -------\" >> $grami_stdout" >> $FBSNG_CLEAN_SCRIPT
	echo "cat ${FBSNG_STDOUT}.${lpid} >> $grami_stdout" >> $FBSNG_CLEAN_SCRIPT
	#echo "echo \"-------- stderr for process #$lpid -------\" >> $grami_stderr" >> $FBSNG_CLEAN_SCRIPT
	echo "cat ${FBSNG_STDERR}.${lpid} >> $grami_stderr" >> $FBSNG_CLEAN_SCRIPT
	echo "rm -f ${FBSNG_STDOUT}.${lpid}" >> $FBSNG_CLEAN_SCRIPT
	echo "rm -f ${FBSNG_STDERR}.${lpid}" >> $FBSNG_CLEAN_SCRIPT
done

debug echo "rm -rf $FBSNG_JOB_DIR"	>> $FBSNG_CLEAN_SCRIPT
debug echo "rm -f $FBSNG_JDF" >> $FBSNG_CLEAN_SCRIPT

chmod +x $FBSNG_CLEAN_SCRIPT

$DEBUG_ECHO "JM_SCRIPT: submitting FBSNG job" >> $grami_logfile

# Execute fbs submit command

#q=`which fbs`
#$DEBUG_ECHO "JM_SCRIPT: ${q}" >> $grami_logfile

$DEBUG_ECHO "JM_SCRIPT: Submit command: fbs submit ${FBSNG_JDF} 2>${FBSNG_JOB_ERR} >${FBSNG_JOB_OUT}" >> $grami_logfile
#myid=`id`
#$DEBUG_ECHO "JM_SCRIPT: Id: $myid" >> $grami_logfile 

status=`fbs submit ${FBSNG_JDF} 2>${FBSNG_JOB_ERR}`
if (test "$?" -eq "0") then
	$DEBUG_ECHO "JM_SCRIPT: awk=${awk}"
	$DEBUG_ECHO "JM_SCRIPT: status=$status"
	job_id=`echo $status | ${awk} '{print $3}'`
	#saving job_id in the arg file for other parts of job run/control
	echo "grami_job_id=${job_id}" >> $arg_file     
	# check if it's numeric
	$DEBUG_ECHO "JM_SCRIPT: job submitted successfully!" >> $grami_logfile
	$DEBUG_ECHO "JM_SCRIPT: returning job state: $GLOBUS_GRAM_PROTOCOL_JOB_STATE_PENDING" >> $grami_logfile
	echo "GRAM_SCRIPT_JOB_ID:$job_id"
	echo "GRAM_SCRIPT_SUCCESS:$GLOBUS_GRAM_PROTOCOL_JOB_STATE_PENDING"
else
   $DEBUG_ECHO "JM_SCRIPT: output from submit command: $status" >> $grami_logfile
   $DEBUG_ECHO "JM_SCRIPT: job *NOT* submitted successfully!" >> $grami_logfile
   echo "GRAM_SCRIPT_ERROR:$GLOBUS_GRAM_PROTOCOL_ERROR_JOB_EXECUTION_FAILED"

   echo ""                                                   >> ${grami_stderr}
   echo "-------- output from scheduler submission --------" >> ${grami_stderr}
   cat ${FBSNG_JOB_OUT}                                      >> ${grami_stderr}
   cat ${FBSNG_JOB_ERR}                                      >> ${grami_stderr}
   echo "--------------------------------------------------" >> ${grami_stderr}
   echo ""                                                   >> ${grami_stderr}
fi

$DEBUG_ECHO "JM_SCRIPT: exiting gram_script_fbsng_submit" >> $grami_logfile

#$DEBUG_ECHO "I found this as stdout:= $grami_stdout " >> $grami_logfile    

exit 1
