#! /bin/sh
#BHEADER**********************************************************************
#
#  Copyright (c) 1995-2009, Lawrence Livermore National Security,
#  LLC. Produced at the Lawrence Livermore National Laboratory. Written
#  by the Parflow Team (see the CONTRIBUTORS file)
#  <parflow@lists.llnl.gov> CODE-OCEC-08-103. All rights reserved.
#
#  This file is part of Parflow. For details, see
#  http://www.llnl.gov/casc/parflow
#
#  Please read the COPYRIGHT file or Our Notice and the LICENSE file
#  for the GNU Lesser General Public License.
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License (as published
#  by the Free Software Foundation) version 2.1 dated February 1999.
#
#  This program is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms
#  and conditions of the GNU General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
#  USA
#**********************************************************************EHEADER
#
# Run parflow in parallel.
# Script hides the different ways to run MPI jobs.
#

#-----------------------------------------------------------------------------
#
# Program usage 
#
#-----------------------------------------------------------------------------
usage () {
   echo "usage: $0 run [-hvpg] pfidb_file num_procs"
   echo "    -h               help"
   echo "    -v               verbose"
   echo "    -p executable    executable to run"
   echo "    -g debug_options cegdb debug optons"
   echo ""
   echo "Runs parflow on using mpi startup command (e.g. mpiexec, mpirun etc)."
}

#=============================================================================
#
# Verbose logging
#
#=============================================================================
pflog () {
   if [ $VERBOSE -eq 1 ]
   then
      echo "pfrun: $@"
   fi
}

if [ -f ~/.parflowrc ] 
then
   . ~/.parflowrc
fi

#=============================================================================
#
# Parse users arguments
#
#=============================================================================
VERBOSE=0
PROGRAM=""
DEBUG=""
while getopts vhpg: c
do
   case $c in
      h)
	 usage;
	 exit 1;;
      v)
	 VERBOSE=1
	 shift;
	 ;;
      p) 
	 PROGRAM=$2
	 shift 
	 shift;;
      g)    for i in $2
	    do 
	       DEBUG="$DEBUG -cegdb $i"
	    done
            shift
	    shift;;
   esac
done

if [ ! \( "$#" -eq 2 \) ]; then
  usage
  exit 1
fi

PFSCRIPT=$1
NUMPROCS=$2

if ! [ -e "${PFSCRIPT}.pfidb" ]; then
  echo "pfidb file \"${PFSCRIPT}.pfidb\" not found" >&2
  exit 1
fi

#
# The following is a hack to deal with NFS temp mount problems
#
if [ -z "$AMPS_ROOT_DIR" ]; then
   AMPS_ROOT_DIR=$HOME
   export AMPS_ROOT_DIR
fi

real_home=`(cd $AMPS_ROOT_DIR; sh -c "pwd")`
AMPS_EXE_DIR=`echo \`pwd\` | sed "s#$real_home#$AMPS_ROOT_DIR#"`
export AMPS_EXE_DIR

if [ -z "$PROGRAM" ]
then
   #
   # The default is to run ParFlow
   #
   PROGRAM=parflow
fi

if [ -z "$DEBUG" ]
then
   DISPLAY_OPT=""
else

   if [ -z "$DISPLAY" ]
   then
      DEBUG_OPT="-display `hostname`:0.0"
   else
      DEBUG_OPT="-display $DISPLAY"
   fi
fi

#
# Specify machine specific ways to launch MPI executables.
#
case `uname -a` in
   *AIX*)
      #
      # IBM SP2
      #
      CMD="poe $PARFLOW_DIR/bin/$PROGRAM -procs `cat .amps.info.${NUMPROCS}` ${PFSCRIPT} $DEBUG_OPT $DEBUG > ${PFSCRIPT}.out.txt"
      ;;
   *WildFire*)
      #
      # For Sun WildFire systems
      #
      pflog "Running with SUN tmrun"
      CMD="tmrun -np `cat .amps.info.${NUMPROCS}`  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} $DEBUG_OPT $DEBUG > ${PFSCRIPT}.out.txt"
      ;;
   *chaos*)
      #
      # For LC chaos systems
      #
      CMD="srun -n ${NUMPROCS} $PARFLOW_RUN_OPTS $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} > ${PFSCRIPT}.out.txt"
      ;;
esac

#
# If mpiexec exists use it first since this is a standard way of running
# mpi executables for MPI-2
#
if which mpiexec >/dev/null 2>&1
then
   CMD="mpiexec -n ${NUMPROCS}  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
elif which mpirun >/dev/null 2>&1
then
   #
   # Didn't have mpiexec so try some mpirun variations
   #
   if [ "`mpirun 2> /dev/null`" = "Missing: program name" ] 
   then
      #
      # Older MPICH. Newer MPICH should be using mpiexec.
      #
      if [ "$PBS_NODEFILE" = "" ]
      then
	 CMD="mpirun -nodes $3 -np ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
      else
	 CMD="mpirun -machinefile $PBS_NODEFILE -nodes $3 -np ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
      fi
   else
      #
      # Other MPI
      #
      CMD="mpirun -np ${NUMPROCS}  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
   fi
elif which aprun >/dev/null 2>&1
then
   #
   # Cray using aprun
   #
   CMD="aprun -n ${NUMPROCS}  $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
elif which srun >/dev/null 2>&1
then
   #
   # Srun based systems
   #
   CMD="srun -n ${NUMPROCS} $PARFLOW_DIR/bin/$PROGRAM ${PFSCRIPT} 2>&1 > ${PFSCRIPT}.out.txt"
else
   echo "Failed to determine how to run mpi executables" >&2
   exit 1
fi

pflog "Running : $CMD" 
eval $CMD
exit $?

