https://gitlab.inria.fr/pm2/pm2
Tip revision: 28348cbb7443cecb94f0380eacf8e9c4fd3b247e authored by Alexandre Denis on 11 July 2022, 16:08:29 UTC
nmad: some more error-checking on remote queues
nmad: some more error-checking on remote queues
Tip revision: 28348cb
mpi_bench_extract.in
#! @BASH@
# -*- mode: Sh;-*-
# MadMPI benchmark
# Copyright (C) 2015-2020 (see AUTHORS file)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
export LC_ALL=C
unset LANGUAGE
unset LANG
out=png
enable_sendrecv=no
enable_rma=no
enable_overlap=no
enable_rmaoverlap=no
enable_colloverlap=no
enable_noise=no
usage() {
echo "# usage: ${0} [--pdf|--png|--svg|--canvas] <file>" >&2
}
while [ "x${1}" != "x" ]; do
case "$1" in
--pdf|--png|--svg|--canvas)
out=$( echo "$1" | sed 's/^--//' )
;;
-h|--help)
usage
exit
;;
-*)
usage
exit 1
;;
*)
file="$1"
;;
esac
shift
done
if [ "x${file}" = "x" ]; then
usage
exit 1
fi
if [ ! -r "${file}" ]; then
echo "# ${0}: cannot read file '${file}'." >&2
usage
exit 1
fi
outdir="${file}.d"
mkdir -p ${outdir}
echo
echo "# ## parsing file ${file}..."
echo
if grep -q '/0 begin' ${file}; then
echo
echo "# ## detected legacy file format; converting..."
sed '/bench:/s,/,%,g' ${file} > ${outdir}/input.dat
file=${outdir}/input.dat
noncontig_suffix=""
echo "# done."
echo
else
noncontig_suffix="%32"
fi
param_delimiter='%'
bench_list_overlap="mpi_bench_overlap_sender mpi_bench_overlap_recv mpi_bench_overlap_bidir mpi_bench_overlap_sender_noncontig mpi_bench_overlap_send_overhead mpi_bench_overlap_Nload"
bench_list_rmaoverlap="mpi_bench_rmaoverlap_put_origin mpi_bench_rmaoverlap_put_target mpi_bench_rmaoverlap_put_passive mpi_bench_rmaoverlap_put_noncontig mpi_bench_rmaoverlap_get_active mpi_bench_rmaoverlap_accumulate"
bench_list_sendrecv="mpi_bench_sendrecv mpi_bench_send_overhead mpi_bench_noncontig${noncontig_suffix} mpi_bench_multi mpi_bench_multi_hetero"
bench_list_rma="mpi_bench_rma_put_active mpi_bench_rma_put_passive mpi_bench_rma_accumulate_active mpi_bench_rma_put_noncontig${param_delimiter}32 mpi_bench_rma_get_active"
bench_list_refrma="mpi_bench_rmaoverlap_accumulate%0 mpi_bench_rmaoverlap_put_origin%0 mpi_bench_rmaoverlap_put_target%0 mpi_bench_rmaoverlap_put_passive%0 mpi_bench_rmaoverlap_get_active%0 mpi_bench_rmaoverlap_put_active%0 mpi_bench_rmaoverlap_put_noncontig%0"
bench_list_reqs="mpi_bench_reqs_burst mpi_bench_reqs_tags mpi_bench_reqs_shuffle mpi_bench_reqs_test mpi_bench_reqs_any"
bench_list_coll="mpi_bench_coll_allgather mpi_bench_coll_allreduce mpi_bench_coll_barrier mpi_bench_coll_bcast mpi_bench_coll_gather mpi_bench_coll_iallgather mpi_bench_coll_iallreduce mpi_bench_coll_igather mpi_bench_coll_ibcast mpi_bench_coll_alltoall mpi_bench_coll_ialltoall"
bench_list_colloverlap="mpi_bench_colloverlap_iallgather mpi_bench_colloverlap_igather mpi_bench_colloverlap_iallreduce mpi_bench_colloverlap_ibcast mpi_bench_colloverlap_ialltoall"
bench_list_refcolloverlap="mpi_bench_colloverlap_iallgather%0 mpi_bench_colloverlap_igather%0 mpi_bench_colloverlap_iallreduce%0 mpi_bench_colloverlap_ibcast%0 mpi_bench_colloverlap_ialltoall%0"
bench_list_noise="mpi_bench_noise_nocomm mpi_bench_noise_omp_nocomm mpi_bench_noise_posted_recv"
echo
echo "# ## generating normalized files..."
echo
detected_bench_list=
for b in ${bench_list_sendrecv} ${bench_list_rma} ${bench_list_refrma} ${bench_list_overlap} ${bench_list_rmaoverlap} ${bench_list_reqs} ${bench_list_coll} ${bench_list_colloverlap} ${bench_list_refcolloverlap} ${bench_list_noise}; do
if grep -q "# bench: ${b} begin" ${file}; then
echo "# processing bench: ${b}"
outfile=${outdir}/${b}.dat
echo "# This file is automatically generated from ${file} for ${b} in machine-readable format." > ${outfile}
echo "# Generated on $( date ) on host $( hostname )" >> ${outfile}
sed -r -f /dev/stdin ${file} >> ${outfile} <<EOF
/${b} begin/,/${b} end/ !d
s/^[ \t]*//g
s/\r//g;s/\t/ /g
/^#/!s/[ \t]+/\t/g
EOF
echo "# ${b}: done."
echo
detected_bench_list="${detected_bench_list} ${b}"
fi
done
if [[ "${detected_bench_list}" =~ mpi_bench_sendrecv ]]; then
enable_sendrecv=yes
expected_bench_list="${expected_bench_list} mpi_bench_sendrecv"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_rma_ ]]; then
enable_rma=yes
expected_bench_list="${expected_bench_list} mpi_bench_rma_put_active mpi_bench_rma_put_passive mpi_bench_rma_get_active mpi_bench_rma_accumulate_active"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_overlap ]]; then
enable_overlap=yes
expected_bench_list="${expected_bench_list} ${bench_list_overlap}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_rmaoverlap ]]; then
enable_rmaoverlap=yes
expected_bench_list="${expected_bench_list} ${bench_list_rmaoverlap}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_coll_ ]]; then
enable_coll=yes
expected_bench_list="${expected_bench_list} ${bench_list_coll}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_colloverlap ]]; then
enable_colloverlap=yes
expected_bench_list="${expected_bench_list} ${bench_list_colloverlap}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_noise ]]; then
enable_noise=yes
expected_bench_list="${expected_bench_list} ${bench_list_noise}"
fi
echo "# "
echo "# detected benchmarks: ${detected_bench_list}"
echo "# "
echo "# enable p2p overlap benchmarks: ${enable_overlap}"
echo "# enable rmaoverlap benchmarks: ${enable_rmaoverlap}"
echo "# enable collective overlap benchmarks: ${enable_colloverlap}"
echo "# enable system noise benchmarks: ${enable_noise}"
echo "# "
echo "# extracting data..."
echo
mpi_bench_extract_line() {
b=$1
s=$2
benchfile=${outdir}/${b}.dat
sed -e "/${b} begin/,/${b} end/ !d;/^#/ d;/^${s}\t/ !d" ${benchfile}
}
mpi_bench_extract_line_param() {
b=$1
s=$2
p=$3
benchfile=${outdir}/${b}.dat
sed -e "/${b}${param_delimiter}${p} begin/,/${b}${param_delimiter}${p} end/ !d;/^#/ d;/^${s}\t/ !d" ${benchfile}
}
mpi_bench_extract_sizes() {
b=$1
sed -e "/^#/ d" ${outdir}/${b}.dat | cut -f 1 | tr '\n' ' '
}
for b in ${expected_bench_list}; do
if [[ ! "${detected_bench_list}" =~ ${b} ]]; then
echo "# ## WARNING- cannot find ${b} in bench file." 1>&2
echo "# detected benchmarks: ${detected_bench_list}" 1>&2
echo
continue
fi
# benchref: reference benchmark without overlap
# lat0: ack latency to substract from rtt (not from benchref)
case ${b} in
mpi_bench_rmaoverlap*)
benchref=${b}%0
lat0=0
;;
mpi_bench_overlap_send_overhead)
benchref=mpi_bench_send_overhead
lat0=0
;;
mpi_bench_overlap_bidir)
benchref=mpi_bench_sendrecv
lat0=0
;;
mpi_bench_overlap*noncontig*)
benchref="mpi_bench_noncontig${noncontig_suffix}"
lat0=$( mpi_bench_extract_line mpi_bench_sendrecv 0 | cut -f 2 )
;;
mpi_bench_overlap_Nload)
benchref=mpi_bench_sendrecv
lat0=0
;;
mpi_bench_overlap_*)
benchref=mpi_bench_sendrecv
lat0=$( mpi_bench_extract_line mpi_bench_sendrecv 0 | cut -f 2 )
;;
mpi_bench_colloverlap_*)
benchref=${b}%0
lat0=0
;;
mpi_bench_*)
benchref="-"
lat0=0
;;
esac
echo "# ## benchmark: ${b}"
benchfile=${outdir}/${b}.dat
echo "# file ${benchfile}"
if [ "x${benchref}" != "x-" ]; then
size_list=$( mpi_bench_extract_sizes ${benchref} )
echo "# sizes: ${size_list}"
params=$( sed -e "s/^# bench: ${b}${param_delimiter}\(.*\) begin/\1/p;d" ${benchfile} | tr '\n' ' ' )
echo "# params list: ${params}"
echo "# generated from ${b} and ${benchref}" > ${outdir}/${b}-ref.dat
echo "# size | comp.time | rtt | lat_ref" >> ${outdir}/${b}-ref.dat
echo -n "# extracting size: "
for s in ${size_list}; do
echo -n "${s} "
latref=$( mpi_bench_extract_line ${benchref} ${s} | cut -f 2 )
mkdir -p ${outdir}/${b}-series
echo "# generated from ${b} for size=${s}" > ${outdir}/${b}-series/${b}-s${s}.dat
echo "# reference latency: ${latref}" >> ${outdir}/${b}-series/${b}-s${s}.dat
echo "# comp.time | ratio | rtt." >> ${outdir}/${b}-series/${b}-s${s}.dat
for p in ${params}; do
rtt=$( mpi_bench_extract_line_param ${b} ${s} ${p} | cut -f 2 )
echo "${s} ${p} ${rtt} ${latref}" >> ${outdir}/${b}-ref.dat
echo "${p} ${ratio} ${rtt}" >> ${outdir}/${b}-series/${b}-s${s}.dat
done
echo >> ${outdir}/${b}-ref.dat
done
else
echo "# no series to extract"
fi
echo
case ${b} in
mpi_bench_overlap_Nload)
gpscript=nload
;;
mpi_bench_overlap*|mpi_bench_rmaoverlap*|mpi_bench_colloverlap*)
gpscript=overlap
;;
mpi_bench_sendrecv|mpi_bench_rma_*|mpi_bench_coll_*|mpi_bench_noise_*)
gpscript=sendrecv
;;
*)
echo "# cannot trace bench: ${b}"
exit 1
;;
esac
cat > ${outdir}/${b}.gp <<EOF
# -*- mode: Gnuplot;-*-
# MadMPI benchmark gnuplot script
# automatically generated by $0 on `date`
# for data set: ${file}; benchmark: ${b}
reset
load "@srcdir@/plot/term-${out}.gp"
benchref = "${benchref}"
lat0 = ${lat0}
call "@srcdir@/plot/trace-${gpscript}.gp" "${file}" "${b}"
EOF
if [ "x@GNUPLOT@" != "x" -a -r ${outdir}/${b}.gp ]; then
echo "# invoking gnuplot ${outdir}/${b}.gp..."
(
cd ${outdir};
@GNUPLOT@ ${b}.gp
)
echo "# done."
else
echo "# gnuplot not detected. Cannot draw graphs."
exit 1
fi
echo
done
if [ ${out} = png ]; then
echo "# ## generating global plot..."
if [ "x@GRAPHICSMAGICK@" = "x" ]; then
echo "# GraphicsMagick not detected. Cannot draw global plots."
exit 1
else
echo "# Using GraphicsMagick:"
@GRAPHICSMAGICK@ version
fi
if [ ${enable_rma} = yes ]; then
echo "# rma base..."
@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - rma latency" $( for b in ${bench_list_rma}; do if [ -s ${outdir}/${b}-lat.png ]; then echo ${outdir}/${b}-lat.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/rma-lat.png
@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - rma bandwidth" $( for b in ${bench_list_rma}; do if [ -s ${outdir}/${b}-bw.png ]; then echo ${outdir}/${b}-bw.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/rma-bw.png
outs="${outs} ${outdir}/rma-lat.png ${outdir}/rma-bw.png"
fi
if [ ${enable_overlap} = yes ]; then
echo "# overlap..."
@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - overlap" $( for b in ${bench_list_overlap}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/overlap.png
outs="${outs} ${outdir}/overlap.png"
fi
if [ ${enable_colloverlap} = yes ]; then
echo "# colloverlap..."
@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - colloverlap" $( for b in ${bench_list_colloverlap}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/colloverlap.png
outs="${outs} ${outdir}/colloverlap.png"
fi
if [ ${enable_rmaoverlap} = yes ]; then
echo "# rmaoverlap..."
@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - rmaoverlap" $( for b in ${bench_list_rmaoverlap}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/rmaoverlap.png
outs="${outs} ${outdir}/rmaoverlap.png"
fi
if [ ${enable_noise} = yes ]; then
echo "# noise..."
@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - noise" $( for b in ${bench_list_noise}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/noise.png
outs="${outs} ${outdir}/noise.png"
fi
if [ "x${outs}" != "x" ]; then
echo "# mosaic..."
@GRAPHICSMAGICK@ convert ${outs} -append ${outdir}/all.png
fi
echo
echo "# ## global plot done."
fi