https://gitlab.inria.fr/pm2/pm2
Raw File
Tip revision: 28348cbb7443cecb94f0380eacf8e9c4fd3b247e authored by Alexandre Denis on 11 July 2022, 16:08:29 UTC
nmad: some more error-checking on remote queues
Tip revision: 28348cb
mpi_bench_extract.in
#! @BASH@
# -*- mode: Sh;-*-

#  MadMPI benchmark
#  Copyright (C) 2015-2020 (see AUTHORS file)
# 
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or (at
#  your option) any later version.
# 
#  This program is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.

export LC_ALL=C
unset LANGUAGE
unset LANG

out=png

enable_sendrecv=no
enable_rma=no
enable_overlap=no
enable_rmaoverlap=no
enable_colloverlap=no
enable_noise=no

usage() {
    echo "# usage: ${0} [--pdf|--png|--svg|--canvas] <file>" >&2
    }

while [ "x${1}" != "x" ]; do
    case "$1" in
	--pdf|--png|--svg|--canvas)
	    out=$( echo "$1" | sed 's/^--//' )
	    ;;
	-h|--help)
	    usage
	    exit
	    ;;
	-*)
	    usage
	    exit 1
	    ;;
	*)
	    file="$1"
	    ;;
    esac
    shift
done

if [ "x${file}" = "x" ]; then
    usage
    exit 1
fi
if [ ! -r "${file}" ]; then
    echo "# ${0}: cannot read file '${file}'." >&2
    usage
    exit 1
fi

outdir="${file}.d"
mkdir -p ${outdir}

echo
echo "# ## parsing file ${file}..."
echo 

if grep -q '/0 begin' ${file}; then
    echo
    echo "# ## detected legacy file format; converting..."
    sed '/bench:/s,/,%,g' ${file} > ${outdir}/input.dat
    file=${outdir}/input.dat
    noncontig_suffix=""
    echo "# done."
    echo
else
    noncontig_suffix="%32"
fi
param_delimiter='%'

bench_list_overlap="mpi_bench_overlap_sender mpi_bench_overlap_recv mpi_bench_overlap_bidir mpi_bench_overlap_sender_noncontig mpi_bench_overlap_send_overhead mpi_bench_overlap_Nload"
bench_list_rmaoverlap="mpi_bench_rmaoverlap_put_origin mpi_bench_rmaoverlap_put_target mpi_bench_rmaoverlap_put_passive mpi_bench_rmaoverlap_put_noncontig mpi_bench_rmaoverlap_get_active mpi_bench_rmaoverlap_accumulate"

bench_list_sendrecv="mpi_bench_sendrecv mpi_bench_send_overhead mpi_bench_noncontig${noncontig_suffix} mpi_bench_multi mpi_bench_multi_hetero"
bench_list_rma="mpi_bench_rma_put_active mpi_bench_rma_put_passive mpi_bench_rma_accumulate_active mpi_bench_rma_put_noncontig${param_delimiter}32 mpi_bench_rma_get_active"
bench_list_refrma="mpi_bench_rmaoverlap_accumulate%0 mpi_bench_rmaoverlap_put_origin%0 mpi_bench_rmaoverlap_put_target%0 mpi_bench_rmaoverlap_put_passive%0 mpi_bench_rmaoverlap_get_active%0 mpi_bench_rmaoverlap_put_active%0 mpi_bench_rmaoverlap_put_noncontig%0"
bench_list_reqs="mpi_bench_reqs_burst mpi_bench_reqs_tags mpi_bench_reqs_shuffle mpi_bench_reqs_test mpi_bench_reqs_any"
bench_list_coll="mpi_bench_coll_allgather mpi_bench_coll_allreduce mpi_bench_coll_barrier mpi_bench_coll_bcast mpi_bench_coll_gather mpi_bench_coll_iallgather mpi_bench_coll_iallreduce mpi_bench_coll_igather mpi_bench_coll_ibcast mpi_bench_coll_alltoall mpi_bench_coll_ialltoall"
bench_list_colloverlap="mpi_bench_colloverlap_iallgather mpi_bench_colloverlap_igather mpi_bench_colloverlap_iallreduce mpi_bench_colloverlap_ibcast mpi_bench_colloverlap_ialltoall"
bench_list_refcolloverlap="mpi_bench_colloverlap_iallgather%0 mpi_bench_colloverlap_igather%0 mpi_bench_colloverlap_iallreduce%0 mpi_bench_colloverlap_ibcast%0 mpi_bench_colloverlap_ialltoall%0"
bench_list_noise="mpi_bench_noise_nocomm mpi_bench_noise_omp_nocomm mpi_bench_noise_posted_recv"

echo
echo "# ## generating normalized files..."
echo
detected_bench_list=
for b in ${bench_list_sendrecv} ${bench_list_rma} ${bench_list_refrma} ${bench_list_overlap} ${bench_list_rmaoverlap} ${bench_list_reqs} ${bench_list_coll} ${bench_list_colloverlap} ${bench_list_refcolloverlap} ${bench_list_noise}; do
    if grep -q "# bench: ${b} begin" ${file}; then
	echo "# processing bench: ${b}"
	outfile=${outdir}/${b}.dat
	echo "# This file is automatically generated from ${file} for ${b} in machine-readable format." > ${outfile}
	echo "# Generated on $( date ) on host $( hostname )" >> ${outfile}
	sed -r -f /dev/stdin ${file} >> ${outfile} <<EOF
/${b} begin/,/${b} end/ !d
s/^[ \t]*//g
s/\r//g;s/\t/ /g
/^#/!s/[ \t]+/\t/g
EOF
	echo "# ${b}: done."
	echo
	detected_bench_list="${detected_bench_list} ${b}"
    fi
done

if [[ "${detected_bench_list}" =~ mpi_bench_sendrecv ]]; then
    enable_sendrecv=yes
    expected_bench_list="${expected_bench_list} mpi_bench_sendrecv"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_rma_ ]]; then
    enable_rma=yes
    expected_bench_list="${expected_bench_list} mpi_bench_rma_put_active mpi_bench_rma_put_passive mpi_bench_rma_get_active mpi_bench_rma_accumulate_active"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_overlap ]]; then
    enable_overlap=yes
    expected_bench_list="${expected_bench_list} ${bench_list_overlap}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_rmaoverlap ]]; then
    enable_rmaoverlap=yes
    expected_bench_list="${expected_bench_list} ${bench_list_rmaoverlap}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_coll_ ]]; then
    enable_coll=yes
    expected_bench_list="${expected_bench_list} ${bench_list_coll}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_colloverlap ]]; then
    enable_colloverlap=yes
    expected_bench_list="${expected_bench_list} ${bench_list_colloverlap}"
fi
if [[ "${detected_bench_list}" =~ mpi_bench_noise ]]; then
    enable_noise=yes
    expected_bench_list="${expected_bench_list} ${bench_list_noise}"
fi

echo "# "
echo "# detected benchmarks: ${detected_bench_list}"
echo "# "
echo "# enable p2p overlap benchmarks:        ${enable_overlap}"
echo "# enable rmaoverlap benchmarks:         ${enable_rmaoverlap}"
echo "# enable collective overlap benchmarks: ${enable_colloverlap}"
echo "# enable system noise benchmarks:       ${enable_noise}"
echo "# "
echo "# extracting data..."
echo

mpi_bench_extract_line() {
    b=$1
    s=$2
    benchfile=${outdir}/${b}.dat
    sed -e "/${b} begin/,/${b} end/ !d;/^#/ d;/^${s}\t/ !d" ${benchfile}
}

mpi_bench_extract_line_param() {
    b=$1
    s=$2
    p=$3
    benchfile=${outdir}/${b}.dat
    sed -e "/${b}${param_delimiter}${p} begin/,/${b}${param_delimiter}${p} end/ !d;/^#/ d;/^${s}\t/ !d" ${benchfile}
}

mpi_bench_extract_sizes() {
    b=$1
    sed -e "/^#/ d" ${outdir}/${b}.dat | cut -f 1 | tr '\n' ' '
}

for b in ${expected_bench_list}; do
    if [[ ! "${detected_bench_list}" =~ ${b} ]]; then
	echo "# ## WARNING- cannot find ${b} in bench file." 1>&2
	echo "# detected benchmarks: ${detected_bench_list}" 1>&2
	echo
	continue
    fi
    # benchref: reference benchmark without overlap
    # lat0: ack latency to substract from rtt (not from benchref)
    case ${b} in
	mpi_bench_rmaoverlap*)
	    benchref=${b}%0
	    lat0=0
	    ;;
	mpi_bench_overlap_send_overhead)
	    benchref=mpi_bench_send_overhead
	    lat0=0
	    ;;
	mpi_bench_overlap_bidir)
	    benchref=mpi_bench_sendrecv
	    lat0=0
	    ;;
	mpi_bench_overlap*noncontig*)
	    benchref="mpi_bench_noncontig${noncontig_suffix}"
	    lat0=$( mpi_bench_extract_line mpi_bench_sendrecv 0 | cut -f 2 )
	    ;;
	mpi_bench_overlap_Nload)
	    benchref=mpi_bench_sendrecv
	    lat0=0
	    ;;
	mpi_bench_overlap_*)
	    benchref=mpi_bench_sendrecv
	    lat0=$( mpi_bench_extract_line mpi_bench_sendrecv 0 | cut -f 2 )
	    ;;
	mpi_bench_colloverlap_*)
	    benchref=${b}%0
	    lat0=0
	    ;;
	mpi_bench_*)
	    benchref="-"
	    lat0=0
	    ;;
    esac
    echo "# ## benchmark: ${b}"
    benchfile=${outdir}/${b}.dat
    echo "# file ${benchfile}"
    if [ "x${benchref}" != "x-" ]; then
	size_list=$( mpi_bench_extract_sizes ${benchref} )
	echo "# sizes: ${size_list}"
	params=$( sed -e "s/^# bench: ${b}${param_delimiter}\(.*\) begin/\1/p;d" ${benchfile} | tr '\n' ' ' )
	echo "# params list: ${params}"
	echo "# generated from ${b} and ${benchref}"       > ${outdir}/${b}-ref.dat
	echo "# size | comp.time | rtt | lat_ref"         >> ${outdir}/${b}-ref.dat
	echo -n "# extracting size: "
	for s in ${size_list}; do
	    echo -n "${s} "
	    latref=$( mpi_bench_extract_line ${benchref} ${s} | cut -f 2 )
	    mkdir -p ${outdir}/${b}-series
	    echo "# generated from ${b} for size=${s}" > ${outdir}/${b}-series/${b}-s${s}.dat
	    echo "# reference latency: ${latref}"     >> ${outdir}/${b}-series/${b}-s${s}.dat
	    echo "# comp.time | ratio | rtt."         >> ${outdir}/${b}-series/${b}-s${s}.dat
	    for p in ${params}; do
		rtt=$( mpi_bench_extract_line_param ${b} ${s} ${p} | cut -f 2 )
		echo "${s} ${p} ${rtt} ${latref}"          >> ${outdir}/${b}-ref.dat
		echo "${p} ${ratio} ${rtt}"                >> ${outdir}/${b}-series/${b}-s${s}.dat
	    done
	    echo  >> ${outdir}/${b}-ref.dat
	done
    else
	echo "# no series to extract"
    fi
    echo

    case ${b} in
	mpi_bench_overlap_Nload)
	    gpscript=nload
	    ;;
	mpi_bench_overlap*|mpi_bench_rmaoverlap*|mpi_bench_colloverlap*)
	    gpscript=overlap
	    ;;
	mpi_bench_sendrecv|mpi_bench_rma_*|mpi_bench_coll_*|mpi_bench_noise_*)
	    gpscript=sendrecv
	    ;;
	*)
	    echo "# cannot trace bench: ${b}"
	    exit 1
	    ;;
    esac

    cat > ${outdir}/${b}.gp <<EOF
# -*- mode: Gnuplot;-*-
# MadMPI benchmark gnuplot script
# automatically generated by $0 on `date`
# for data set: ${file}; benchmark: ${b}

reset

load "@srcdir@/plot/term-${out}.gp"
benchref = "${benchref}"
lat0 = ${lat0}
call "@srcdir@/plot/trace-${gpscript}.gp" "${file}" "${b}"

EOF
    if [ "x@GNUPLOT@" != "x" -a -r ${outdir}/${b}.gp ]; then
	echo "# invoking gnuplot ${outdir}/${b}.gp..."
	(
	    cd ${outdir};
	    @GNUPLOT@ ${b}.gp
	)
	echo "# done."
    else
	echo "# gnuplot not detected. Cannot draw graphs."
	exit 1
    fi
    echo
done

if [ ${out} = png ]; then
    echo "# ## generating global plot..."
    if [ "x@GRAPHICSMAGICK@" = "x" ]; then
	echo "# GraphicsMagick not detected. Cannot draw global plots."
	exit 1
    else
	echo "# Using GraphicsMagick:"
	@GRAPHICSMAGICK@ version
    fi
    if [ ${enable_rma} = yes ]; then
	echo "# rma base..."
	@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - rma latency" $( for b in ${bench_list_rma}; do if [ -s ${outdir}/${b}-lat.png ]; then echo ${outdir}/${b}-lat.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/rma-lat.png
	@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - rma bandwidth" $( for b in ${bench_list_rma}; do if [ -s ${outdir}/${b}-bw.png ]; then echo ${outdir}/${b}-bw.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/rma-bw.png
	outs="${outs} ${outdir}/rma-lat.png ${outdir}/rma-bw.png"
    fi
    if [ ${enable_overlap} = yes ]; then
	echo "# overlap..."
	@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - overlap" $( for b in ${bench_list_overlap}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/overlap.png
	outs="${outs} ${outdir}/overlap.png"
    fi
    if [ ${enable_colloverlap} = yes ]; then
	echo "# colloverlap..."
	@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - colloverlap" $( for b in ${bench_list_colloverlap}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/colloverlap.png
	outs="${outs} ${outdir}/colloverlap.png"
    fi
    if [ ${enable_rmaoverlap} = yes ]; then
	echo "# rmaoverlap..."
	@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - rmaoverlap" $( for b in ${bench_list_rmaoverlap}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/rmaoverlap.png
	outs="${outs} ${outdir}/rmaoverlap.png"
    fi
    if [ ${enable_noise} = yes ]; then
	echo "# noise..."
	@GRAPHICSMAGICK@ convert -size 800x768 label:"${file}\n - noise" $( for b in ${bench_list_noise}; do if [ -s ${outdir}/${b}.png ]; then echo ${outdir}/${b}.png; else echo "-size 1024x768 label:${b}"; fi; done ) +append ${outdir}/noise.png
	outs="${outs} ${outdir}/noise.png"
    fi
    if [ "x${outs}" != "x" ]; then
	echo "# mosaic..."
	@GRAPHICSMAGICK@ convert ${outs} -append ${outdir}/all.png
    fi
    echo
    echo "# ## global plot done."
fi

back to top