#!/bin/bash
# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
set -e

exec > >(tee submit.log)

# Without this, only stdout would be captured - i.e. your
# log file would not contain any error messages.
# SEE answer by Adam Spiers, which keeps STDERR a seperate stream -
# I did not want to steal from him by simply adding his answer to mine.
exec 2>&1

DAX_FILE=""
CACHE_FILE=""
LOCAL_PEGASUS_DIR=""
ACCOUNTING_GROUP=""
NO_ACCOUNTING_GROUP=0

GETOPT_CMD=`getopt -o d:c:a:Fl:h --long dax:,cache-file:,accounting-group:,force-no-accounting-group,local-dir:,help -n 'pycbc_submit_dax' -- "$@"`
eval set -- "$GETOPT_CMD"

while true ; do
  case "$1" in
    -d|--dax)
      case "$2" in
        "") shift 2 ;;
        *) DAX_FILE=$2 ; shift 2 ;;
      esac ;;
    -c|--cache-file)
      case "$2" in
        "") shift 2 ;;
        *) CACHE_FILE=$2 ; shift 2 ;;
      esac ;;
    -a|--accounting-group)
      case "$2" in
        "") shift 2 ;;
        *) ACCOUNTING_GROUP=$2 ; shift 2 ;;
      esac ;;
    -F|--force-no-accounting-group) NO_ACCOUNTING_GROUP=1 ; shift ;;
    -l|--local-dir)
      case "$2" in
        "") shift 2 ;;
        *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;;
      esac ;;
    -h|--help)
      echo "usage: pycbc_submit_dax [-h]"
      echo "                        --dax DAX"
      echo "                        [--cache-file FILE]"
      echo "                        [--accounting-group TAG]"
      echo "                        [--force-no-accounting-group]"
      echo "                        [--local-dir DIR]"
      echo
      echo "required arguments:"
      echo "  -d, --dax DAX           name of the dax file to plan"
      echo
      echo "optional arguments:"
      echo "  -h, --help              show this help message and exit"
      echo "  -c, --cache-file FILE   replica cache file for data reuse"
      echo "  -a, --accounting-group GROUP tagged string used for site "
      echo "                               resource accounting."
      echo "  -F, --force-no-accounting-group submit without an accounting"
      echo "                               group. Will cause condor submission"
      echo "                               to fail on LIGO Data Grid clusters"
      echo "  -l, --local-dir         Directory to put condor files under. "
      echo
      echo "If the environment variable TMPDIR is set then this is prepended to the "
      echo "path to the temporary workflow execte directory passed to pegasus-plan."
      echo "If the --local-dir option is not given."
      echo
      exit 0 ;;
    --) shift ; break ;;
    *) echo "Internal error!" ; exit 1 ;;
  esac
done

if [ "x$DAX_FILE" == "x" ]; then
  echo "Error: --dax must be specified. Use --help for options."
   exit 1
fi

if [ "x$ACCOUNTING_GROUP" == "x" ] && [ $NO_ACCOUNTING_GROUP == 0 ]; then
  echo "Error: You must specify and accounting group with --accounting-group or"
  echo "override this check with --force-no-accounting-group. If you do not specify"
  echo "an accounting group, job submission will fail on the LIGO Data Grid."
  echo
  echo "For a list of available LIGO Data Grid accounting group tags, see"
  echo "   https://ldas-gridmon.ligo.caltech.edu/ldg_accounting/user"
  exit 1
fi

#Make a directory for the submit files
SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX`

#Make sure the directory is world readable
chmod 755 $SUBMIT_DIR

# find the site-local template directory
PEG_DIR=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print PEGASUS_FILE_DIRECTORY'`
SITE_TEMPLATE=$PEG_DIR/basic_site_template.xml

export ACCOUNTING_GROUP

# Set up the site-local with the correct paths
echo 'cat <<END_OF_TEXT' >  temp.sh
cat $SITE_TEMPLATE       >> temp.sh
echo 'END_OF_TEXT'       >> temp.sh
bash temp.sh > site-local-parsed.xml

# Plan the workflow
echo "Generating concrete workflow"
touch _reuse.cache

if [ "x$CACHE_FILE" == "x" ]; then
  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $DAX_FILE --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --relative-submit-dir work --cluster label --submit 
else
  cp $CACHE_FILE _reuse.cache
  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $DAX_FILE --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --cache _reuse.cache --relative-submit-dir work --cluster label --submit
fi

ln -s $SUBMIT_DIR submitdir

echo pegasus-status $SUBMIT_DIR/work > status
chmod 755 status

echo pegasus-analyzer $SUBMIT_DIR/work > debug
chmod 755 debug

echo pegasus-remove $SUBMIT_DIR/work > stop
chmod 755 stop

echo pegasus-run $SUBMIT_DIR/work > start
chmod 755 start

echo Submitting from: $HOSTNAME