#!/bin/csh
#PBS -l walltime=6:00:00
#PBS -l rmsnodes=1:4
#PBS -j oe

#
#  This script requires exactly same directory structure both for
#  $HOME/$WORKDIR and $SCRATCH/$WORKDIR
#
#
set echo

# -------- NOTE TO  MFIX USERS 
# Variables (i.e. path, filename, user e-mail address etc) 
# that need to be changed for launching a job are
# shown between two dashed lines. PLEASE DO NOT CHANGE other
# parts of the script 
#
# ============================================================
#
# Set the run name
setenv RUNNAME "CFB3D"
# Set the working directory where mfix.dat file will be copied from 
# e.g. /usr/users/4/gel/MFIX/RUN
setenv WORKDIR mfix/3dCfb

# Set the default e-mail
setenv USEREMAIL "USERNAME@psc.edu"

# Set the path to the executable
setenv EXEPATH ~USERNAME/mfix/

# Set the filename for the executable
setenv MFIXEXEFILE "mfix.exe"

# =============================================================
# DO NOT CHANGE ANYTHING BELOW UNLESS YOU KNOW WHAT YOU ARE
# DOING.

#echo 'Local directory is :' $LOCAL

if(-e $HOME/$WORKDIR/RUNCONTROL ) then
       echo "RUNCONTROL found"
else
     # Create a RUNCONTROL file for a new start 
       set RUN_NO = 1 
       echo y > $HOME/$WORKDIR/RUNCONTROL
       echo $RUN_NO >> $HOME/$WORKDIR/RUNCONTROL
endif

# Read in the header information from RUNCONTROL and set
# flag variable to determine if another job will be 
# submitted or not from this one  
set CONTINUERUN = `head -n 1 $HOME/$WORKDIR/RUNCONTROL` 

#
# if the first line in RUNCONTROL parameter file
# is set to 'y' or 'Y' (Yes) then continue with
# tasks for running the job 
if ( $CONTINUERUN == "y" || $CONTINUERUN == "Y" ) then 

   #  Determine the number for current RUN which will be 
   #  used in copying appropriate mfix.dat file for this
   #  run (mfix.dat_R1,....,mfix.dat_R3 etc.)
   set RUN_NO = `tail -n 1 $HOME/$WORKDIR/RUNCONTROL` 

   echo $RUN_NO
   #
   # copy input file from HOME to SCRATCH
   if(-e $HOME/$WORKDIR/mfix.dat_R$RUN_NO ) then
      cp -f -r $HOME/$WORKDIR/mfix.dat_R$RUN_NO $SCRATCH/$WORKDIR/mfix.dat
   else
      echo "mfix.dat for Run $RUN_NO NOT FOUND in $HOME/$WORKDIR" 
      mail $USEREMAIL <<EOF
        There is a problem with this job. 
        mfix.dat for Run $RUN_NO NOT FOUND in $HOME/$WORKDIR 
EOF
      exit
   endif

   #
   # copy the executable from HOME location to SCRATCH workdir
   if(-e $EXEPATH/$MFIXEXEFILE ) then
      cp -f -r $EXEPATH/$MFIXEXEFILE $SCRATCH/$WORKDIR
   else
      echo "$MFIXEXEFILE NOT FOUND in $EXEPATH"
      mail $USEREMAIL <<EOF
        There is a problem with this job.
        $MFIXEXEFILE NOT FOUND in $EXEPATH
EOF
      exit
   endif

   cd $SCRATCH/$WORKDIR 
   chmod u+x  $MFIXEXEFILE
   # check the path
   pwd
   ls -als 

   # execute your program
   # Run option 1
   prun -s -t -N ${RMS_NODES} -n ${RMS_PROCS} ./$MFIXEXEFILE
   # Run option 2
   #prun -s -t -N ${RMS_NODES} -n ${RMS_PROCS} '{compute}:$LOCAL/mfix.exe'
   # Run option 3 with debug details
   #prun -s -t -vvv -N ${RMS_NODES} -n ${RMS_PROCS} ./$MFIXEXEFILE


#DBG
   set flag = $status
#  set flag = 0
   echo "prun return value is $flag "

   if ($flag != 0 ) then
        mail $USEREMAIL <<EOF
        There is a problem with this job.  prun mfix.exe failed.
EOF
        exit
   endif


   # copy output files from $LOCAL to $STAGE
   # delete all files on $LOCAL
   # see section on tcscp and tcsrm for an explantion
   # of the following lines
   #tcscp -nk '{compute}:$LOCAL/PSDF*' $WORKDIR
   #tcsrm '{compute}:$LOCAL/*'

   echo 'work directory contents at the end of job'
   ls -als
#   far mkdir $WORKDIR/$RUN_NO
#   far store $RUNNAME.* $WORKDIR/$RUN_NO 
#   far store $RUNNAME???.LOG $WORKDIR/$RUN_NO 
#   far store mfix.dat $WORKDIR/$RUN_NO

   # Set the run counter and write into RUNCONTROL file for next job submission
   set RUN_NO = `expr $RUN_NO + 1`
   set CHKCONTINUERUN = `head -n 1 $HOME/$WORKDIR/RUNCONTROL` 
   echo $CHKCONTINUERUN > $HOME/$WORKDIR/RUNCONTROL
   echo $RUN_NO >> $HOME/$WORKDIR/RUNCONTROL


#DBG

endif

END:
#echo 'work directory contents at the end of job'
#ls -als

exit