https://github.com/gwastro/pycbc
Tip revision: db928bb4c20f71ba0eb69e9a77334ae184397634 authored by Gareth S Davies on 27 July 2020, 09:55:02 UTC
Prep for release (#3396)
Prep for release (#3396)
Tip revision: db928bb
pycbc_submit_dax
#!/bin/bash
# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
set -e
mkdir -p workflow/planning
COMMAND_LINE="${0} ${@}"
echo "${COMMAND_LINE}" > workflow/planning/pycbc-submit-dax-command.txt
echo "Executing on `hostname` in directory `pwd` at `date`" > workflow/planning/pycbc-submit-dax-log.txt
exec > >(tee -a workflow/pycbc-submit-dax-log.txt)
# Without this, only stdout would be captured - i.e. your
# log file would not contain any error messages.
# SEE answer by Adam Spiers, which keeps STDERR a seperate stream -
# I did not want to steal from him by simply adding his answer to mine.
exec 2>&1
DAX_FILE=""
CACHE_FILE=0
LOCAL_PEGASUS_DIR=""
ACCOUNTING_GROUP=""
ACCOUNTING_GROUP_USER=""
NO_ACCOUNTING_GROUP=0
PEGASUS_PROPERTIES=""
SHARED_FILESYSTEM=0
SITE_LIST=""
STAGING_SITES=""
TRANSFORMATION_CATALOG=""
LOCAL_STAGING_SERVER="file://"
REMOTE_STAGING_SERVER=""
NO_CREATE_PROXY=0
NO_GRID=""
SUBMIT_DAX="--submit"
HTML_ENTITIES="{\"\'\": ''', '(': '(', ')': ')', '+': '+', '\"': '"'}"
# These will be changed by the bundle builder
DATA_INLINE=False
function expand_pegasus_files() {
(
base64 -d <<DATA_END
PEGASUS_FILE_DATA
DATA_END
) | tar -zx
}
echo "# Properties set on command line" > extra-properties.conf
rm -f _reuse.cache
touch _reuse.cache
rm -f *-extra-site-properties.xml
GETOPT_CMD=`getopt -o d:c:g:r:a:u:p:P:es:S:k:t:Fknl:Gh --long dax:,cache-file:,local-staging-server:,remote-staging-server:,accounting-group:,accounting-group-user:,pegasus-properties:,append-pegasus-property:,enable-shared-filesystem,execution-sites:,staging-sites:,append-site-profile:,transformation-catalog:,force-no-accounting-group,no-create-proxy,no-submit,local-dir:,no-grid,help -n 'pycbc_submit_dax' -- "$@"`
eval set -- "$GETOPT_CMD"
while true ; do
case "$1" in
-d|--dax)
case "$2" in
"") shift 2 ;;
*) DAX_FILE=$2 ; shift 2 ;;
esac ;;
-c|--cache-file)
case "$2" in
"") shift 2 ;;
*) CACHE_FILE=1
OLD_IFS=${IFS}
IFS=","
set +e
for URL in $2; do
FOUND_URL=1
# strip of any leading file:// URI and see if the file exists
FILE_URL=${URL##file://}
if [ -f ${FILE_URL} ] ; then
cat ${FILE_URL} >> _reuse.cache
FOUND_URL=0
else
# otherwise try and get the file using resolve_url
DOWNLOADED_FILE=`python - <<EOF
from pycbc.workflow import resolve_url
import os
filename = resolve_url("${FILE_URL}")
print(filename)
EOF`
FOUND_URL=${?}
if [ ${FOUND_URL} -eq 0 ] ; then
cat $DOWNLOADED_FILE >> _reuse.cache
rm -f $DOWNLOADED_FILE
fi
fi
if [ ${FOUND_URL} -ne 0 ] ; then
echo "$URL not found!"
exit
fi
IFS=${OLD_IFS}
done
set -e
shift 2 ;;
esac ;;
-g|--local-staging-server)
case "$2" in
"") shift 2 ;;
*) LOCAL_STAGING_SERVER=$2 ; shift 2 ;;
esac ;;
-r|--remote-staging-server)
case "$2" in
"") shift 2 ;;
*) REMOTE_STAGING_SERVER=$2 ; shift 2 ;;
esac ;;
-a|--accounting-group)
case "$2" in
"") shift 2 ;;
*) export ACCOUNTING_GROUP=$2 ; shift 2 ;;
esac ;;
-u|--accounting-group-user)
case "$2" in
"") shift 2 ;;
*) export ACCOUNTING_GROUP_USER=$2 ; shift 2 ;;
esac ;;
-p|--pegasus-properties)
case "$2" in
"") shift 2 ;;
*) PEGASUS_PROPERTIES=$2 ; shift 2 ;;
esac ;;
-P|--append-pegasus-property)
case "$2" in
"") shift 2 ;;
*) echo $2 >> extra-properties.conf ; shift 2 ;;
esac ;;
-e|--enable-shared-filesystem) SHARED_FILESYSTEM=1 ; shift ;;
-s|--execution-sites)
case "$2" in
"") shift 2 ;;
*) SITE_LIST=$2 ; shift 2 ;;
esac ;;
-S|--staging-sites)
case "$2" in
"") shift 2 ;;
*) STAGING_SITES="--staging-site $2" ; shift 2 ;;
esac ;;
-k|--append-site-profile)
case "$2" in
"") shift 2 ;;
*) SITE_PROFILE=${2}
APPEND_SITE_PROFILE_ARRAY=$(python -c "import sys; from xml.sax.saxutils import (escape, unescape); print(escape(unescape(sys.stdin.read()), entities=${HTML_ENTITIES}))" <<< ${SITE_PROFILE})
OIFS=${IFS}; IFS=":"
read -ra APPEND_SITE_PROFILE_ARRAY <<< "${APPEND_SITE_PROFILE_ARRAY}"
IFS=${OIFS}
APPEND_SITE=${APPEND_SITE_PROFILE_ARRAY[0]}
APPEND_SITE_NS_KEY=(${APPEND_SITE_PROFILE_ARRAY[1]//\|/ })
APPEND_SITE_VALUE=("${APPEND_SITE_PROFILE_ARRAY[@]:2}")
/bin/echo -n " <profile namespace=\"${APPEND_SITE_NS_KEY[0]}\" key=\"${APPEND_SITE_NS_KEY[1]}\">" >> ${APPEND_SITE}-extra-site-properties.xml
need_colon=0
for i in "${APPEND_SITE_VALUE[@]}" ; do
if [ ${need_colon} -eq 1 ] ; then
/bin/echo -n ":" >> ${APPEND_SITE}-extra-site-properties.xml
fi
/bin/echo -n "${i}" >> ${APPEND_SITE}-extra-site-properties.xml
need_colon=1
done
echo "</profile>" >> ${APPEND_SITE}-extra-site-properties.xml
shift 2 ;;
esac ;;
-t|--transformation-catalog)
case "$2" in
"") shift 2 ;;
*) TRANSFORMATION_CATALOG=$2 ; shift 2 ;;
esac ;;
-F|--force-no-accounting-group) NO_ACCOUNTING_GROUP=1 ; shift ;;
-K|--no-create-proxy) NO_CREATE_PROXY=1 ; shift ;;
-n|--no-submit) SUBMIT_DAX="" ; shift ;;
-l|--local-dir)
case "$2" in
"") shift 2 ;;
*) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;;
esac ;;
-G|--no-grid) NO_GRID="--forward nogrid" ; shift ;;
-h|--help)
echo "usage: pycbc_submit_dax [-h] --dax DAX [optional arguments]"
echo
echo "required arguments:"
echo " -d, --dax DAX name of the dax file to plan"
echo
echo "workflow submission required one of:"
echo " -a, --accounting-group GROUP tagged string used for site "
echo " resource accounting."
echo " -F, --force-no-accounting-group submit without an accounting"
echo " group. Will cause condor submission"
echo " to fail on LIGO Data Grid clusters"
echo
echo "optional arguments:"
echo " -h, --help show this help message and exit"
echo " -c, --cache-file FILE replica cache file for data reuse"
echo " -g, --local-staging-server uri://host provide a uri on host"
echo " for the local site scratch"
echo " (default is file://)"
echo " -r, --remote-staging-server uri://host provide a uri on host"
echo " for the remote site scratch."
echo " -p, --pegasus-properties FILE use the specified file as"
echo " the pegasus properties file"
echo " -P, --append-pegasus-property STRING add the extra property"
echo " specified by the argument"
echo " -s, --execution-sites A,B,C specify a comma separated list"
echo " of execution sites that will be"
echo " used in addition to the local site"
echo " -S, --staging-sites A=X,B=Y,C=Z comma separated list of key=value"
echo " pairs, where the key is the"
echo " execution site and value is the"
echo " staging site for that execution site"
echo " -k, --append-site-profile SITE:NAMESPACE|KEY:VALUE"
echo " append the profile determined by"
echo " NAMESPACE, KEY, and VALUE to the"
echo " site catalog entry for SITE"
echo " -t, --transformation-catalog FILE pass the specified"
echo " transformation catalog to Pegasus"
echo " -K, --no-create-proxy Do not run ligo-proxy-init and assume"
echo " that the user has a valid grid proxy"
echo " -n, --no-submit Plan the DAX but do not submit it"
echo " -l, --local-dir Directory to put condor files under"
echo " -G, --no-grid Disable checks for grid proxy and"
echo " GLOBUS_LOCATION in pegasus-plan"
echo
echo "If the environment variable TMPDIR is set then this is prepended to the "
echo "path to the temporary workflow execute directory passed to pegasus-plan."
echo "If the --local-dir option is not given."
echo
echo "If the environment variable PEGASUS_FILE_DIRECTORY is set then the"
echo "script will look there for pegasus site catalog and configuration"
echo "otherwise, the script will look for this directory by querying the"
echo "pycbc.workflow module."
echo
echo "If the environment variable PEGASUS_SITE_CATALOG_PATH is set then the"
echo "script will look for site catalog templates in this directory."
echo "Site catalog templates may contain other environment variables that"
echo "must be set in order for them to be rendered correctly."
echo
exit 0 ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
if [ "x$DAX_FILE" == "x" ]; then
echo "Error: --dax must be specified. Use --help for options."
exit 1
fi
if [ "x$ACCOUNTING_GROUP" == "x" ] && [ $NO_ACCOUNTING_GROUP == 0 ]; then
echo "Error: You must specify an accounting group with --accounting-group or"
echo "override this check with --force-no-accounting-group. If you do not specify"
echo "an accounting group, job submission will fail on the LIGO Data Grid."
echo
echo "For a list of available LIGO Data Grid accounting group tags, see"
echo " https://ldas-gridmon.ligo.caltech.edu/ldg_accounting/user"
exit 1
fi
if [ $NO_CREATE_PROXY == 0 ]; then
# Force the user to create a new grid proxy
LIGO_USER_NAME=""
while true; do
read -p "Enter your LIGO.ORG username in (e.g. albert.einstein): " LIGO_USER_NAME
echo
if [ ! -z $LIGO_USER_NAME ] ; then
break
fi
done
unset X509_USER_PROXY
ligo-proxy-init -p $LIGO_USER_NAME || exit 1
else
if [ ! -z ${X509_USER_PROXY} ] ; then
if [ -f ${X509_USER_PROXY} ] ; then
cp -a ${X509_USER_PROXY} /tmp/x509up_u`id -u`
fi
unset X509_USER_PROXY
fi
fi
if [ -z "${NO_GRID}" ] ; then
#Check that the proxy is valid
set +e
grid-proxy-info -exists
RESULT=$?
set -e
if [ ${RESULT} -eq 0 ] ; then
PROXY_TYPE=`grid-proxy-info -type | tr -d ' '`
if [ x${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
grid-proxy-info
else
cp /tmp/x509up_u`id -u` /tmp/x509up_u`id -u`.orig
grid-proxy-init -cert /tmp/x509up_u`id -u`.orig -key /tmp/x509up_u`id -u`.orig
rm -f /tmp/x509up_u`id -u`.orig
grid-proxy-info
fi
else
echo "Error: Could not find a valid grid proxy to submit workflow."
exit 1
fi
fi
#Make a directory for the submit files
SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX`
#Make sure the directory is world readable
chmod 755 $SUBMIT_DIR
# find the site-local template directory
if [ $DATA_INLINE == "True" ] ; then
expand_pegasus_files
PEGASUS_FILE_DIRECTORY=${PWD}/pegasus_files
elif [ -z $PEGASUS_FILE_DIRECTORY ] ; then
PEGASUS_FILE_DIRECTORY=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print(PEGASUS_FILE_DIRECTORY)'`
fi
# Create the site catalog
export LOCAL_SITE_PATH=${PWD}
LOCAL_SITE_STORAGE_PATH=${LOCAL_SITE_PATH}
LOCAL_SITE_STORAGE_URL='file://'${LOCAL_SITE_PATH}
if [[ "$LOCAL_STAGING_SERVER" == stash://* ]] ; then
LOCAL_SITE_SCRATCH_PATH=${LOCAL_SITE_PATH}
LOCAL_SITE_SCRATCH_URL='stash://'`echo ${LOCAL_SITE_SCRATCH_PATH} | sed 's+^/stash++g'`
else
LOCAL_SITE_SCRATCH_PATH=${LOCAL_SITE_PATH}
LOCAL_SITE_SCRATCH_URL=${LOCAL_STAGING_SERVER}${LOCAL_SITE_PATH}
fi
export LOCAL_SITE_SCRATCH_PATH LOCAL_SITE_SCRATCH_URL
export LOCAL_SITE_STORAGE_PATH LOCAL_SITE_STORAGE_URL
# add the local site
# Set remote staging url to local
if [ -z $SITE_LIST ] ; then
SITE_LIST="local"
else
SITE_LIST="local,${SITE_LIST}"
fi
if [ -z ${PEGASUS_SITE_CATALOG_PATH} ] ; then
PEGASUS_SITE_CATALOG_PATH=${PEGASUS_FILE_DIRECTORY}
fi
# Create the site catalog
# Don't overwrite remote staging URL if site list is local
if [ ${SITE_LIST} != "local" ] ; then
if [ -z $REMOTE_STAGING_SERVER ] ; then
REMOTE_SITE_SCRATCH_URL="REMOTE_STAGING_SERVER_UNSET" # this will be used as an error flag
else
if [[ "${REMOTE_STAGING_SERVER}" == stash://* ]] ; then
REMOTE_SITE_SCRATCH_PATH=${LOCAL_SITE_PATH}
REMOTE_SITE_SCRATCH_URL='stash://'`echo ${REMOTE_SITE_SCRATCH_PATH} | sed 's+^/stash++g'`
else
REMOTE_SITE_SCRATCH_PATH=${LOCAL_SITE_PATH}
REMOTE_SITE_SCRATCH_URL="${REMOTE_STAGING_SERVER}${REMOTE_SITE_SCRATCH_PATH}"
fi
fi
fi
export REMOTE_SITE_SCRATCH_PATH REMOTE_SITE_SCRATCH_URL
# create the site catalog template
if [ $NO_ACCOUNTING_GROUP == 0 ] ; then
cat >> local-extra-site-properties.xml << EOF
<profile namespace="condor" key="accounting_group">$ACCOUNTING_GROUP</profile>
<profile namespace="condor" key="accounting_group_user">$ACCOUNTING_GROUP_USER</profile>
EOF
fi
if [ -z "${NO_GRID}" ] ; then
echo ' <profile namespace="condor" key="use_x509userproxy">True</profile>' >> local-extra-site-properties.xml
fi
echo 'cat <<END_OF_TEXT' > site-catalog.sh
cat << EOF >> site-catalog.sh
<?xml version="1.0" encoding="UTF-8"?>
<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd" version="4.0">
EOF
OLD_IFS=${IFS}
IFS=','
read -ra SITE <<< "$SITE_LIST"
IFS=${OLD_IFS}
for s in "${SITE[@]}"; do
SITE_TEMPLATE=$PEGASUS_SITE_CATALOG_PATH/${s}-site-template.xml
if [ ! -e ${SITE_TEMPLATE} ] ; then
echo "Error: Could not find site catalog template for site ${s}"
exit 1
else
cat $SITE_TEMPLATE >> site-catalog.sh
if [ -e ${s}-extra-site-properties.xml ] ; then
cat ${s}-extra-site-properties.xml >> site-catalog.sh
fi
echo " </site>" >> site-catalog.sh
fi
done
cat << EOF >> site-catalog.sh
</sitecatalog>
EOF
echo 'END_OF_TEXT' >> site-catalog.sh
# write out the site catalog
bash site-catalog.sh > site-catalog.xml
# If REMOTE_STAGING_SERVER was not set, but was needed by
# sites the user asked for, then raise an error and exit
if grep -q REMOTE_STAGING_SERVER_UNSET site-catalog.xml ; then
echo "Attempt to use a remote site but --remote-staging-server not specified"
exit 1
fi
# Plan the workflow
echo "Generating concrete workflow"
# cache the pegasus config
if [ -z ${PEGASUS_PROPERTIES} ] ; then
cp $PEGASUS_FILE_DIRECTORY/pegasus-properties.conf ./pegasus-properties.conf
else
cp ${PEGASUS_PROPERTIES} ./pegasus-properties.conf
fi
# add a transformation catalog, if specified
if [ ! -z ${TRANSFORMATION_CATALOG} ] ; then
cp ${TRANSFORMATION_CATALOG} ./transformation-catalog.txt
echo >> pegasus-properties.conf
echo "pegasus.catalog.transformation Text" >> pegasus-properties.conf
echo "pegasus.catalog.transformation.file ${PWD}/transformation-catalog.txt" >> pegasus-properties.conf
fi
echo >> pegasus-properties.conf
cat extra-properties.conf >> pegasus-properties.conf
# If user specified a value for pegasus.data.configuration
# then don't add another one
if ! grep -q pegasus.data.configuration= pegasus-properties.conf
then
# No option specified, append nonshared unless user said
# not to
if [ $SHARED_FILESYSTEM == 0 ]
then
echo 'pegasus.data.configuration=nonsharedfs' >> pegasus-properties.conf
fi
fi
if [ $CACHE_FILE == 0 ]; then
pegasus-plan --conf ./pegasus-properties.conf -d $DAX_FILE --sites $SITE_LIST $STAGING_SITES -o local --dir $SUBMIT_DIR --cleanup inplace --relative-dir work --cluster horizontal $SUBMIT_DAX $NO_GRID
else
pegasus-plan --conf ./pegasus-properties.conf -d $DAX_FILE --sites $SITE_LIST $STAGING_SITES -o local --dir $SUBMIT_DIR --cleanup inplace --cache _reuse.cache --relative-dir work --cluster horizontal $SUBMIT_DAX $NO_GRID
fi
echo
rm -f submitdir
ln -sf $SUBMIT_DIR submitdir
echo "pegasus-status --verbose --long $SUBMIT_DIR/work \$@" > status
chmod 755 status
echo "pegasus-analyzer -r -v $SUBMIT_DIR/work \$@" > debug
chmod 755 debug
echo "pegasus-remove $SUBMIT_DIR/work \$@" > stop
chmod 755 stop
if [ -z "${NO_GRID}" ] ; then
cat << EOF > start
#!/bin/bash
if [ -f /tmp/x509up_u\`id -u\` ] ; then
unset X509_USER_PROXY
else
if [ ! -z \${X509_USER_PROXY} ] ; then
if [ -f \${X509_USER_PROXY} ] ; then
cp -a \${X509_USER_PROXY} /tmp/x509up_u\`id -u\`
fi
fi
unset X509_USER_PROXY
fi
# Check that the proxy is valid
grid-proxy-info -exists
RESULT=\${?}
if [ \${RESULT} -eq 0 ] ; then
PROXY_TYPE=\`grid-proxy-info -type | tr -d ' '\`
if [ x\${PROXY_TYPE} == 'xRFC3820compliantimpersonationproxy' ] ; then
grid-proxy-info
else
cp /tmp/x509up_u\`id -u\` /tmp/x509up_u\`id -u\`.orig
grid-proxy-init -cert /tmp/x509up_u\`id -u\`.orig -key /tmp/x509up_u\`id -u\`.orig
rm -f /tmp/x509up_u\`id -u\`.orig
grid-proxy-info
fi
else
echo "Error: Could not find a valid grid proxy to submit workflow."
exit 1
fi
EOF
fi
echo "pegasus-run $SUBMIT_DIR/work \$@" > start
chmod 755 start
# Copy planning information into workflow directory so it can be displayed on the results page
cp pegasus-properties.conf workflow/planning
cp site-catalog.xml workflow/planning
cp $SUBMIT_DIR/work/braindump.txt workflow/planning
mkdir -p workflow/input_map
cp _reuse.cache workflow/input_map/reuse.cache
mkdir -p workflow/dax
mkdir -p workflow/output_map
# copy the main dax and output.map to the workflow directory
cp $DAX_FILE workflow/dax
if [ -e output.map ] ; then
cp output.map workflow/output_map
fi
# ugly shell to extract the sub-dax names from the uberdax xml
for dax in `egrep '<[[:blank:]]*dax' ${DAX_FILE} | tr ' ' \\\n | grep file | awk -F= '{print $2}' | tr -d '">'` ; do
if [ -e ${dax} ] ; then
cp ${dax} workflow/dax
fi
map=`basename ${dax} .dax`.map
if [ -e ${map} ] ; then
cp ${map} workflow/output_map
fi
done
if [ -z ${SUBMIT_DAX} ] ; then
echo
echo "WARNING: DAX planned but not submitted. No dashboard entry has been created and"
echo " the workflow section of results page will not show a dashboard URL."
echo " You must run this script without the --no-submit option if this is a"
echo " production run."
echo
exit 0
fi
if [ ! -e ${HOME}/.pegasus/workflow.db ] ; then
echo "WARNING: Could not find Pegasus dashboard database in ${HOME}/.pegasus"
echo " Workflow has been submitted but the results page will not contain"
echo " a link to the dashboard page. If this is a production workflow,"
echo " please remove the workflow, check for the origin of this error,"
echo " and re-submit the workflow by re-running this script."
echo
exit 1
fi
WORKFLOW_ID_STRING=""
WORKFLOW_DB_CMD="sqlite3 -csv ${HOME}/.pegasus/workflow.db \"select submit_hostname,wf_id,wf_uuid from master_workflow where submit_dir = '${SUBMIT_DIR}/work';\""
DB_TRY=0
# force a condor reschedule to get the workflow running
set +e
condor_reschedule &> /dev/null
set -e
/bin/echo "Querying Pegasus database for workflow stored in ${SUBMIT_DIR}/work"
/bin/echo -n "This may take up to 120 seconds. Please wait..."
rm -f pegasus_db.log
touch pegasus_db.log
# querying the database sometimes fails, so allow retries
set +e
until [ $DB_TRY -ge 15 ]
do
/bin/echo -n "."
WORKFLOW_ID_STRING=`eval $WORKFLOW_DB_CMD 2>> pegasus_db.log`
if [ $? -eq 0 ] && [ ! -z $WORKFLOW_ID_STRING ] ; then
/bin/echo " Done."
DB_QUERY_SUCCESS=0
break
else
DB_QUERY_SUCCESS=1
fi
DB_TRY=$(( $DB_TRY + 1 ))
for s in `seq ${DB_TRY}`
do
/bin/echo -n "."
sleep 1
done
done
set -e
if [ ${DB_QUERY_SUCCESS} -eq 1 ] ; then
echo; echo
/bin/echo "Query failed: ${WORKFLOW_DB_CMD}"
cat pegasus_db.log
else
rm -f pegasus_db.log
fi
if [ -z $WORKFLOW_ID_STRING ] ; then
echo "WARNING: Could not find the workflow in the Pegasus dashboard database."
echo " Workflow has been submitted but the results page will not contain"
echo " a link to the dashboard page. If this is a production workflow,"
echo " please remove the workflow, check for the origin of this error,"
echo " and re-submit the workflow by re-running this script."
echo
exit 1
fi
WORKFLOW_ID_ARRAY=(${WORKFLOW_ID_STRING//,/ })
DASHBOARD_URL="https://${WORKFLOW_ID_ARRAY[0]}/pegasus/u/${USER}/r/${WORKFLOW_ID_ARRAY[1]}/w?wf_uuid=${WORKFLOW_ID_ARRAY[2]}"
echo ${DASHBOARD_URL} > workflow/pegasus-dashboard-url.txt
shopt -s nullglob
DASHBOARD_GLOB=(results/*_workflow/*DASHBOARD*.html)
DASHBOARD_PATH=${DASHBOARD_GLOB[0]}
if [ -e ${DASHBOARD_PATH} ] ; then
DASHBOARD_COPY=workflow/`basename ${DASHBOARD_PATH}`.orig
if [ -e ${DASHBOARD_COPY} ] ; then
cp ${DASHBOARD_COPY} ${DASHBOARD_PATH}
else
cp ${DASHBOARD_PATH} ${DASHBOARD_COPY}
fi
COMMAND_LINE=$(python -c "import sys; from xml.sax.saxutils import (escape, unescape); print(escape(unescape(sys.stdin.read()), entities=${HTML_ENTITIES}))" <<< ${SITE_PROFILE})
sed -i.bak -e "s+PYCBC_SUBMIT_DAX_ARGV+${COMMAND_LINE}+g" ${DASHBOARD_PATH}
sed -i.bak -e "s+PEGASUS_DASHBOARD_URL+${DASHBOARD_URL}+g" ${DASHBOARD_PATH}
fi
shopt -u nullglob
echo "Workflow submission completed successfully."
echo
echo "The Pegasus dashboard URL for this workflow is:"
echo " ${DASHBOARD_URL}"
echo
echo "Note that it make take a while for the dashboard entry to appear while the workflow"
echo "is parsed by the dashboard. The delay can be on the order of one hour for very large"
echo "workflows."
echo
# If the workflow results directory exists, copy the files there
for dir in planning dax input_map output_map
do
RESULT_DIR=results/*_workflow/*_${dir}
if [ -d ${RESULT_DIR} ] ; then
cp workflow/${dir}/* ${RESULT_DIR}
fi
done
exit 0