https://github.com/chu11/magpie
Tip revision: 219c5e875fe515d051dd0032a6f05eaade734e26 authored by Albert Chu on 29 October 2014, 22:47:05 UTC
Fix invalid build/release
Fix invalid build/release
Tip revision: 219c5e8
magpie-run-spark-sparkwordcount
#!/bin/bash
#############################################################################
# Copyright (C) 2013 Lawrence Livermore National Security, LLC.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Albert Chu <chu11@llnl.gov>
# LLNL-CODE-644248
#
# This file is part of Magpie, scripts for running Hadoop on
# traditional HPC systems. For details, see <URL>.
#
# Magpie is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# Magpie is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Magpie. If not, see <http://www.gnu.org/licenses/>.
#############################################################################
# This script is the core sparkwordcount running script. For the most
# part, it shouldn't be editted. See job submission files for
# configuration details.
source ${MAGPIE_SCRIPTS_HOME}/magpie-common-exports
source ${MAGPIE_SCRIPTS_HOME}/magpie-common-functions
sparkwordcountclass="org.apache.spark.examples.JavaWordCount"
cd ${SPARK_HOME}
# For some reason, the examples jar is named with seemingly random
# versioning, so we'll have to "find" it for each case
if [ "${SPARK_SPARKWORDCOUNT_COPY_IN_FILE}X" != "X" ]
then
if echo ${SPARK_SPARKWORDCOUNT_FILE} | grep -q -E "hdfs:\/\/"
then
# Make directory of destination file
filedir=`dirname ${SPARK_SPARKWORDCOUNT_FILE}`
command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -mkdir -p ${filedir}"
echo "Running $command" >&2
$command
if echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^file:\/\/"
then
command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -copyFromLocal ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
elif echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^hdfs:\/\/"
then
command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -cp ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
fi
elif echo ${SPARK_SPARKWORDCOUNT_FILE} | grep -q -E "file:\/\/"
then
# Make directory of destination file
filedir=`dirname ${SPARK_SPARKWORDCOUNT_FILE}`
filedir=`echo ${filedir} | sed "s/^file:\/\///"`
command="mkdir -p ${filedir}"
echo "Running $command" >&2
$command
if echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^file:\/\/"
then
command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -cp ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
elif echo ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} | grep -q -E "^hdfs:\/\/"
then
command="${HADOOP_HOME}/${hadoopcmdprefix}/hadoop fs -cpToLocal ${SPARK_SPARKWORDCOUNT_COPY_IN_FILE} ${SPARK_SPARKWORDCOUNT_FILE}"
fi
fi
echo "Running copy in $command" >&2
$command
fi
if echo ${SPARK_VERSION} | grep -q -E "1\.[0-9]\.[0-9]"
then
sparkexamplesjar=`find ${SPARK_HOME}/lib | grep jar | grep examples`
command="${sparkcmdprefix}/spark-submit --class ${sparkwordcountclass} ${sparkexamplesjar} ${SPARK_SPARKWORDCOUNT_FILE}"
else
sparkexamplesjar=`find ${SPARK_HOME}/examples | grep jar | grep examples | grep -v assembly | grep -v sources`
# Ugh, Spark 0.9.1 doesn't have a way to pass in specific jars
# except via the SPARK_CLASSPATH environment variable.
#
# Because user may have set SPARK_JOB_CLASSPATH, we can't just set
# it here. So we'll reput it into spark-env.sh
if [ "${SPARK_JOB_CLASSPATH}X" != "X" ]
then
echo "export SPARK_CLASSPATH=\"${SPARK_JOB_CLASSPATH};${sparkexamplesjar}\"" >> ${SPARK_CONF_DIR}/spark-env.sh
else
echo "export SPARK_CLASSPATH=\"${sparkexamplesjar}\"" >> ${SPARK_CONF_DIR}/spark-env.sh
fi
command="${sparkcmdprefix}/spark-class ${sparkwordcountclass} spark://$SPARK_MASTER_NODE:$SPARK_MASTER_PORT ${SPARK_SPARKWORDCOUNT_FILE}"
fi
echo "Running $command" >&2
$command
exit 0