Re: [RFC PATCH] benchtests/bench-plot.py: Add graphing script for string benchmarks.

[Siddhesh Poyarekar <siddhesh at redhat dot com>]

On Wed, Sep 04, 2013 at 03:49:56PM +0100, Will Newton wrote:
> 
> Add a Python script that uses pylab to graph the results of string
> benchmarks. Currently it only supports graphing the results of the
> strlen and memcpy benchmarks.

Sorry Will, I had this marked for review and I completely forgot about
it.  The script doesn't seem to work with the current files, probably
because the -ifunc tests were merged in.  Would you be able to post an
updated version?

I am OK with having a python script to do this; in fact I worked on
porting my benchmark parsing script (which I'll post soon) since I
found it easier to write a more readable program compared to perl.
However, I believe there were reservations in the past about
introducing additional dependencies, so I'm wondering if we could make
an exception here and add a dependency on python for running
benchmarks.  I'll start a discussion on that in a separate thread when
I post my ported script.

Thanks,
Siddhesh

> 
> ChangeLog:
> 
> 2013-09-03   Will Newton  <will.newton@linaro.org>
> 
> 	* benchtests/bench-plot.py: New file.
> ---
>  benchtests/bench-plot.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 184 insertions(+)
>  create mode 100755 benchtests/bench-plot.py
> 
> diff --git a/benchtests/bench-plot.py b/benchtests/bench-plot.py
> new file mode 100755
> index 0000000..7fd680b
> --- /dev/null
> +++ b/benchtests/bench-plot.py
> @@ -0,0 +1,184 @@
> +#!/usr/bin/env python
> +# Copyright (C) 2013 Free Software Foundation, Inc.
> +# This file is part of the GNU C Library.
> +
> +# The GNU C Library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +
> +# The GNU C Library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with the GNU C Library; if not, see
> +# <http://www.gnu.org/licenses/>.
> +
> +# A script for graphing the results of glibc string benchmarks.
> +
> +import glob
> +import math
> +import os
> +import re
> +import sys
> +
> +import pylab
> +
> +def unique_values(rows, column):
> +    values = []
> +    for row in rows:
> +        if not row[column] in values:
> +            values.append(row[column])
> +    return sorted(values)
> +
> +def make_colours():
> +    return iter('m b g r c y k pink orange brown grey'.split())
> +
> +def pretty_kb(v):
> +    if v < 1024:
> +        return '%d' % v
> +    else:
> +        if v % 1024 == 0:
> +            return '%d k' % (v//1024)
> +        else:
> +            return '%.1f k' % (v/1024)
> +
> +def plot_null(benchmark, input_files, oldinput_files, output_path):
> +    pass
> +
> +def parse_benchmark_rows(input_files):
> +    impls = []
> +    rows = []
> +    matcher = re.compile("Length\s+(\d+), alignment\s+([0-9 /]+):")
> +    for input_file in input_files:
> +        lines = open(input_file).readlines()
> +        if not impls:
> +            impls = lines[0].strip().split("\t")
> +        for line in lines[1:]:
> +            columns = line.split("\t")
> +            m = matcher.match(columns[0])
> +            groups = m.groups()
> +            length = groups[0]
> +            alignment = groups[1]
> +            rows.append([int(length), alignment] + map(float, columns[1:]))
> +    return (impls, rows)
> +
> +def plot_benchmark(benchmark, input_files, oldinput_files, output_path):
> +    (newimpls, newrows) = parse_benchmark_rows(input_files)
> +    (oldimpls, oldrows) = parse_benchmark_rows(oldinput_files)
> +    # We're only interested in the glibc impl for the old files
> +    if oldimpls:
> +        oldimpls = ["%s (old)" % oldimpls[0]]
> +    alignments = unique_values(newrows, 1)
> +    for alignment in alignments:
> +        # Draw one figure per alignment value
> +        pylab.figure(1).set_size_inches((12, 10))
> +        pylab.clf()
> +        plot_done = False
> +        colours = make_colours()
> +        for (impls, rows) in ((newimpls, newrows), (oldimpls, oldrows)):
> +            if not rows:
> +                continue
> +            plot_rows = []
> +            for row in rows:
> +                if row[1] == alignment:
> +                    plot_rows.append(row)
> +            X = unique_values(plot_rows, 0)
> +            # Filter out zero length entries
> +            X = [x for x in X if x > 0]
> +            # If there are too few data points, skip this alignment
> +            if len(X) < 2:
> +                continue
> +            numimpls = len(impls)
> +            Y = []
> +            Yerr = []
> +            # Initialize the Y and Yerr arrays
> +            for i in range(0, numimpls):
> +                Y.append([])
> +                Yerr.append([[], []])
> +            for length in X:
> +                matches = [x for x in plot_rows if x[0] == length]
> +                # If there is more than one test run for a given test
> +                # then calculate the mean and min/max
> +                if len(matches) > 1:
> +                    for i in range(0, numimpls):
> +                        vals = [x[2 + i] for x in matches]
> +                        mean = length / (sum(vals)/len(vals))
> +                        Y[i].append(mean)
> +                        err1 = (length / max(vals)) - mean
> +                        if err1 < 0:
> +                            err1 = 0
> +                        err2 = (length / min(vals)) - mean
> +                        if err2 > 0:
> +                            err2 = 0
> +                        Yerr[i][0].append(abs(err2))
> +                        Yerr[i][1].append(err1)
> +                else:
> +                    for i in range(0, numimpls):
> +                        Y[i].append(float(length) / matches[2 + i])
> +            i = 0
> +            for impl in impls:
> +                colour = colours.next()
> +                pylab.plot(X, Y[i], c=colour)
> +                plot_done = True
> +                # If we have error values then draw the bars
> +                if len(Yerr[i]) > 0:
> +                    pylab.errorbar(X, Y[i], yerr=Yerr[i], c=colour, label=impl,
> +                                   fmt='o')
> +                else:
> +                    pylab.scatter(X, Y[i], c=colour, label=impl,
> +                                  edgecolors='none')
> +                i += 1
> +        # If we didn't draw anything then skip this graph
> +        if not plot_done:
> +            continue
> +        pylab.legend(loc='upper left', ncol=3, prop={'size': 'small'})
> +        pylab.grid()
> +        # Tidy up alignment text for benchmarks that need it
> +        alignment = alignment.replace(" ", "")
> +        alignment = alignment.replace("/", "-")
> +        pylab.title('%s of %s byte aligned buffers' % (benchmark, alignment))
> +        pylab.xlabel('Length (B)')
> +        pylab.ylabel('Bytes per cycle or ns')
> +
> +        top = max(X)
> +
> +        power = int(round(math.log(top) / math.log(2)))
> +
> +        pylab.semilogx()
> +
> +        pylab.axes().set_xticks([2**x for x in range(0, power+1)])
> +        pylab.axes().set_xticklabels([pretty_kb(2**x)
> +                                      for x in range(0, power+1)])
> +        pylab.xlim(0, top)
> +        pylab.ylim(0, pylab.ylim()[1])
> +        pylab.savefig(os.path.join(output_path, '%s-%s.png' %
> +                                   (benchmark, alignment)), dpi=72)
> +
> +plotters = {
> +    "memcpy" : plot_benchmark,
> +    "strlen" : plot_benchmark,
> +}
> +
> +def plot_benchmark(directory, benchmark):
> +    plotter = plotters.get(benchmark, plot_null)
> +    input_path = os.path.join(directory, "bench-%s.*.out" % benchmark)
> +    input_files = glob.glob(input_path)
> +    if not input_files:
> +        return
> +    oldinput_path = os.path.join(directory, "bench-%s.*.out.old" % benchmark)
> +    oldinput_files = glob.glob(oldinput_path)
> +    plotter(benchmark, input_files, oldinput_files, directory)
> +
> +def usage():
> +    print "bench-plot.py <directory> <benchmark>"
> +    sys.exit(1)
> +
> +if __name__ == '__main__':
> +    if len(sys.argv) != 3:
> +        usage()
> +    directory = sys.argv[1]
> +    benchmark = sys.argv[2]
> +    plot_benchmark(directory, benchmark)
> -- 
> 1.8.1.4
>