#! /bin/sh
#BHEADER**********************************************************************
#
#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
#  LLC. Produced at the Lawrence Livermore National Laboratory. Written
#  by the Parflow Team (see the CONTRIBUTORS file)
#  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
#
#  This file is part of Parflow. For details, see
#  http://www.llnl.gov/casc/parflow
#
#  Please read the COPYRIGHT file or Our Notice and the LICENSE file
#  for the GNU Lesser General Public License.
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License (as published
#  by the Free Software Foundation) version 2.1 dated February 1999.
#
#  This program is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
#  and conditions of the GNU General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
#  USA
#**********************************************************************EHEADER
#
# Run parflow in parallel.
# Script hides the different ways to run MPI jobs.
#

#-----------------------------------------------------------------------------
#
# Program usage 
#
#-----------------------------------------------------------------------------
usage () {
   echo "usage: $0 run [-hvpg] pfidb_file num_procs"
   echo "    -h               help"
   echo "    -v               verbose"
   echo "    -p executable    executable to run"
   echo "    -g debug_options cegdb debug optons"
   echo ""
   echo "Runs parflow on using mpi startup command (e.g. mpiexec, mpirun etc)."
}

#=============================================================================
#
# Verbose logging
#
#=============================================================================
pflog () {
   if [ $VERBOSE -eq 1 ]
   then
      echo "pfrun: $@"
   fi
}

if [ -f ~/.parflowrc ] 
then
   . ~/.parflowrc
fi

#=============================================================================
#
# Parse users arguments
#
#=============================================================================
VERBOSE=0
PROGRAM=""
DEBUG=""
module load mpi
MPIEXEC="mpiexec"
while getopts vmhpg: c
do
   case $c in
      h)
	 usage;
	 exit 1;;
      v)
	 VERBOSE=1
	 shift;
	 ;;
      p) 
	 PROGRAM=$2
	 shift 
	 shift;;
      g)    for i in $2
	    do 
	       DEBUG="$DEBUG -cegdb $i"
	    done
            shift
	    shift;;
   esac
done

if [ ! \( "$#" -eq 2 \) ]; then
  usage
  exit 1
fi

PFSCRIPT=$1
NUMPROCS=$2

if ! [ -e "${PFSCRIPT}.pfidb" ]; then
  echo "pfidb file \"${PFSCRIPT}.pfidb\" not found" >&2
  exit 1
fi

#
# The following is a hack to deal with NFS temp mount problems
#
if [ -z "$AMPS_ROOT_DIR" ]; then
   AMPS_ROOT_DIR=$HOME
   export AMPS_ROOT_DIR
fi

real_home=`(cd $AMPS_ROOT_DIR; sh -c "pwd")`
AMPS_EXE_DIR=`echo \`pwd\` | sed "s#$real_home#$AMPS_ROOT_DIR#"`
export AMPS_EXE_DIR

if [ -z "$PROGRAM" ]
then
   #
   # The default is to run ParFlow
   #
   PROGRAM=parflow
fi

if [ -z "$DEBUG" ]
then
   DISPLAY_OPT=""
else

   if [ -z "$DISPLAY" ]
   then
      DEBUG_OPT="-display `hostname`:0.0"
   else
      DEBUG_OPT="-display $DISPLAY"
   fi
fi

CMD=""

# If CMake was used then use setting from file for MPI commands
if [ -f $PARFLOW_DIR/config/pf-cmake-env.sh ]
then
   . $PARFLOW_DIR/config/pf-cmake-env.sh

   if [ -z ${PARFLOW_MEMORYCHECK_COMMAND} ]
   then
      CMD="${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_POSTFLAGS} ${PARFLOW_MPIEXEC_EXTRA_FLAGS} ${MPIEXEC_PREFLAGS} ${PARFLOW_DIR}/bin/${PROGRAM} ${PFSCRIPT} $DEBUG_OPT $DEBUG 2>&1 > ${PFSCRIPT}.out.txt"
   else
      CMD="${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_POSTFLAGS} ${PARFLOW_MPIEXEC_EXTRA_FLAGS} ${MPIEXEC_PREFLAGS} ${PARFLOW_MEMORYCHECK_COMMAND} ${PARFLOW_MEMORYCHECK_COMMAND_OPTIONS} ${PARFLOW_DIR}/bin/${PROGRAM} ${PFSCRIPT} $DEBUG_OPT $DEBUG 2>&1 > ${PFSCRIPT}.out.txt"
   fi
fi

#
# Specify machine specific ways to launch MPI executables.
#
if [ -z "$CMD" ]
then
   case `uname -a` in
      *Linux*)
         CMD="mpiexec --allow-run-as-root -np ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} $DEBUG_OPT $DEBUG > ${PFSCRIPT}.out.txt"
         ;;
      *AIX*)
	 #
	 # IBM SP2
	 #
	 CMD="poe $PARFLOW_DIR/bin/$PROGRAM -procs `cat .amps.info.${NUMPROCS}` ${PFSCRIPT} $DEBUG_OPT $DEBUG > ${PFSCRIPT}.out.txt"
	 ;;
      *WildFire*)
	 #
	 # For Sun WildFire systems
	 #
	 pflog "Running with SUN tmrun"
	 CMD="tmrun -np `cat .amps.info.${NUMPROCS}`  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} $DEBUG_OPT $DEBUG > ${PFSCRIPT}.out.txt"
	 ;;
      *chaos*)
	 #
	 # For LC chaos systems
	 #
	 CMD="srun -n ${NUMPROCS} $PARFLOW_RUN_OPTS $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} > ${PFSCRIPT}.out.txt"
      ;;
   esac
fi

if [ -z "$CMD" ]
then
   #
   # If mpiexec exists use it first since this is a standard way of running
   # mpi executables for MPI-2
   #
   if which mpiexec >/dev/null 2>&1
   then
      CMD="mpiexec -n ${NUMPROCS}  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
   elif which mpirun >/dev/null 2>&1
   then
      #
      # Didn't have mpiexec so try some mpirun variations
      #
      if [ "`mpirun 2> /dev/null`" = "Missing: program name" ] 
      then
	 #
	 # Older MPICH. Newer MPICH should be using mpiexec.
	 #
	 if [ "$PBS_NODEFILE" = "" ]
	 then
	    CMD="mpirun -nodes $3 -np ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
	 else
	    CMD="mpirun -machinefile $PBS_NODEFILE -nodes $3 -np ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
	 fi
      else
	 #
	 # Other MPI
	 #
	 CMD="mpirun -np ${NUMPROCS}  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
      fi
   elif which srun >/dev/null 2>&1
   then
      #
      # Srun based systems
      #
      CMD="srun -n ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
   else
      echo "Failed to determine how to run mpi executables" >&2
      exit 1
   fi
fi

pflog "executing command : $CMD" 
eval $CMD
exit $?

