| Message ID | 52274894.4000604@linaro.org |
|---|---|
| State | Rejected |
| Headers | show |
On Wed, Sep 04, 2013 at 03:49:56PM +0100, Will Newton wrote: > > Add a Python script that uses pylab to graph the results of string > benchmarks. Currently it only supports graphing the results of the > strlen and memcpy benchmarks. Sorry Will, I had this marked for review and I completely forgot about it. The script doesn't seem to work with the current files, probably because the -ifunc tests were merged in. Would you be able to post an updated version? I am OK with having a python script to do this; in fact I worked on porting my benchmark parsing script (which I'll post soon) since I found it easier to write a more readable program compared to perl. However, I believe there were reservations in the past about introducing additional dependencies, so I'm wondering if we could make an exception here and add a dependency on python for running benchmarks. I'll start a discussion on that in a separate thread when I post my ported script. Thanks, Siddhesh > > ChangeLog: > > 2013-09-03 Will Newton <will.newton@linaro.org> > > * benchtests/bench-plot.py: New file. > --- > benchtests/bench-plot.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 184 insertions(+) > create mode 100755 benchtests/bench-plot.py > > diff --git a/benchtests/bench-plot.py b/benchtests/bench-plot.py > new file mode 100755 > index 0000000..7fd680b > --- /dev/null > +++ b/benchtests/bench-plot.py > @@ -0,0 +1,184 @@ > +#!/usr/bin/env python > +# Copyright (C) 2013 Free Software Foundation, Inc. > +# This file is part of the GNU C Library. > + > +# The GNU C Library is free software; you can redistribute it and/or > +# modify it under the terms of the GNU Lesser General Public > +# License as published by the Free Software Foundation; either > +# version 2.1 of the License, or (at your option) any later version. > + > +# The GNU C Library is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > +# Lesser General Public License for more details. > + > +# You should have received a copy of the GNU Lesser General Public > +# License along with the GNU C Library; if not, see > +# <http://www.gnu.org/licenses/>. > + > +# A script for graphing the results of glibc string benchmarks. > + > +import glob > +import math > +import os > +import re > +import sys > + > +import pylab > + > +def unique_values(rows, column): > + values = [] > + for row in rows: > + if not row[column] in values: > + values.append(row[column]) > + return sorted(values) > + > +def make_colours(): > + return iter('m b g r c y k pink orange brown grey'.split()) > + > +def pretty_kb(v): > + if v < 1024: > + return '%d' % v > + else: > + if v % 1024 == 0: > + return '%d k' % (v//1024) > + else: > + return '%.1f k' % (v/1024) > + > +def plot_null(benchmark, input_files, oldinput_files, output_path): > + pass > + > +def parse_benchmark_rows(input_files): > + impls = [] > + rows = [] > + matcher = re.compile("Length\s+(\d+), alignment\s+([0-9 /]+):") > + for input_file in input_files: > + lines = open(input_file).readlines() > + if not impls: > + impls = lines[0].strip().split("\t") > + for line in lines[1:]: > + columns = line.split("\t") > + m = matcher.match(columns[0]) > + groups = m.groups() > + length = groups[0] > + alignment = groups[1] > + rows.append([int(length), alignment] + map(float, columns[1:])) > + return (impls, rows) > + > +def plot_benchmark(benchmark, input_files, oldinput_files, output_path): > + (newimpls, newrows) = parse_benchmark_rows(input_files) > + (oldimpls, oldrows) = parse_benchmark_rows(oldinput_files) > + # We're only interested in the glibc impl for the old files > + if oldimpls: > + oldimpls = ["%s (old)" % oldimpls[0]] > + alignments = unique_values(newrows, 1) > + for alignment in alignments: > + # Draw one figure per alignment value > + pylab.figure(1).set_size_inches((12, 10)) > + pylab.clf() > + plot_done = False > + colours = make_colours() > + for (impls, rows) in ((newimpls, newrows), (oldimpls, oldrows)): > + if not rows: > + continue > + plot_rows = [] > + for row in rows: > + if row[1] == alignment: > + plot_rows.append(row) > + X = unique_values(plot_rows, 0) > + # Filter out zero length entries > + X = [x for x in X if x > 0] > + # If there are too few data points, skip this alignment > + if len(X) < 2: > + continue > + numimpls = len(impls) > + Y = [] > + Yerr = [] > + # Initialize the Y and Yerr arrays > + for i in range(0, numimpls): > + Y.append([]) > + Yerr.append([[], []]) > + for length in X: > + matches = [x for x in plot_rows if x[0] == length] > + # If there is more than one test run for a given test > + # then calculate the mean and min/max > + if len(matches) > 1: > + for i in range(0, numimpls): > + vals = [x[2 + i] for x in matches] > + mean = length / (sum(vals)/len(vals)) > + Y[i].append(mean) > + err1 = (length / max(vals)) - mean > + if err1 < 0: > + err1 = 0 > + err2 = (length / min(vals)) - mean > + if err2 > 0: > + err2 = 0 > + Yerr[i][0].append(abs(err2)) > + Yerr[i][1].append(err1) > + else: > + for i in range(0, numimpls): > + Y[i].append(float(length) / matches[2 + i]) > + i = 0 > + for impl in impls: > + colour = colours.next() > + pylab.plot(X, Y[i], c=colour) > + plot_done = True > + # If we have error values then draw the bars > + if len(Yerr[i]) > 0: > + pylab.errorbar(X, Y[i], yerr=Yerr[i], c=colour, label=impl, > + fmt='o') > + else: > + pylab.scatter(X, Y[i], c=colour, label=impl, > + edgecolors='none') > + i += 1 > + # If we didn't draw anything then skip this graph > + if not plot_done: > + continue > + pylab.legend(loc='upper left', ncol=3, prop={'size': 'small'}) > + pylab.grid() > + # Tidy up alignment text for benchmarks that need it > + alignment = alignment.replace(" ", "") > + alignment = alignment.replace("/", "-") > + pylab.title('%s of %s byte aligned buffers' % (benchmark, alignment)) > + pylab.xlabel('Length (B)') > + pylab.ylabel('Bytes per cycle or ns') > + > + top = max(X) > + > + power = int(round(math.log(top) / math.log(2))) > + > + pylab.semilogx() > + > + pylab.axes().set_xticks([2**x for x in range(0, power+1)]) > + pylab.axes().set_xticklabels([pretty_kb(2**x) > + for x in range(0, power+1)]) > + pylab.xlim(0, top) > + pylab.ylim(0, pylab.ylim()[1]) > + pylab.savefig(os.path.join(output_path, '%s-%s.png' % > + (benchmark, alignment)), dpi=72) > + > +plotters = { > + "memcpy" : plot_benchmark, > + "strlen" : plot_benchmark, > +} > + > +def plot_benchmark(directory, benchmark): > + plotter = plotters.get(benchmark, plot_null) > + input_path = os.path.join(directory, "bench-%s.*.out" % benchmark) > + input_files = glob.glob(input_path) > + if not input_files: > + return > + oldinput_path = os.path.join(directory, "bench-%s.*.out.old" % benchmark) > + oldinput_files = glob.glob(oldinput_path) > + plotter(benchmark, input_files, oldinput_files, directory) > + > +def usage(): > + print "bench-plot.py <directory> <benchmark>" > + sys.exit(1) > + > +if __name__ == '__main__': > + if len(sys.argv) != 3: > + usage() > + directory = sys.argv[1] > + benchmark = sys.argv[2] > + plot_benchmark(directory, benchmark) > -- > 1.8.1.4 >
diff --git a/benchtests/bench-plot.py b/benchtests/bench-plot.py new file mode 100755 index 0000000..7fd680b --- /dev/null +++ b/benchtests/bench-plot.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python +# Copyright (C) 2013 Free Software Foundation, Inc. +# This file is part of the GNU C Library. + +# The GNU C Library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. + +# The GNU C Library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. + +# You should have received a copy of the GNU Lesser General Public +# License along with the GNU C Library; if not, see +# <http://www.gnu.org/licenses/>. + +# A script for graphing the results of glibc string benchmarks. + +import glob +import math +import os +import re +import sys + +import pylab + +def unique_values(rows, column): + values = [] + for row in rows: + if not row[column] in values: + values.append(row[column]) + return sorted(values) + +def make_colours(): + return iter('m b g r c y k pink orange brown grey'.split()) + +def pretty_kb(v): + if v < 1024: + return '%d' % v + else: + if v % 1024 == 0: + return '%d k' % (v//1024) + else: + return '%.1f k' % (v/1024) + +def plot_null(benchmark, input_files, oldinput_files, output_path): + pass + +def parse_benchmark_rows(input_files): + impls = [] + rows = [] + matcher = re.compile("Length\s+(\d+), alignment\s+([0-9 /]+):") + for input_file in input_files: + lines = open(input_file).readlines() + if not impls: + impls = lines[0].strip().split("\t") + for line in lines[1:]: + columns = line.split("\t") + m = matcher.match(columns[0]) + groups = m.groups() + length = groups[0] + alignment = groups[1] + rows.append([int(length), alignment] + map(float, columns[1:])) + return (impls, rows) + +def plot_benchmark(benchmark, input_files, oldinput_files, output_path): + (newimpls, newrows) = parse_benchmark_rows(input_files) + (oldimpls, oldrows) = parse_benchmark_rows(oldinput_files) + # We're only interested in the glibc impl for the old files + if oldimpls: + oldimpls = ["%s (old)" % oldimpls[0]] + alignments = unique_values(newrows, 1) + for alignment in alignments: + # Draw one figure per alignment value + pylab.figure(1).set_size_inches((12, 10)) + pylab.clf() + plot_done = False + colours = make_colours() + for (impls, rows) in ((newimpls, newrows), (oldimpls, oldrows)): + if not rows: + continue + plot_rows = [] + for row in rows: + if row[1] == alignment: + plot_rows.append(row) + X = unique_values(plot_rows, 0) + # Filter out zero length entries + X = [x for x in X if x > 0] + # If there are too few data points, skip this alignment + if len(X) < 2: + continue + numimpls = len(impls) + Y = [] + Yerr = [] + # Initialize the Y and Yerr arrays + for i in range(0, numimpls): + Y.append([]) + Yerr.append([[], []]) + for length in X: + matches = [x for x in plot_rows if x[0] == length] + # If there is more than one test run for a given test + # then calculate the mean and min/max + if len(matches) > 1: + for i in range(0, numimpls): + vals = [x[2 + i] for x in matches] + mean = length / (sum(vals)/len(vals)) + Y[i].append(mean) + err1 = (length / max(vals)) - mean + if err1 < 0: + err1 = 0 + err2 = (length / min(vals)) - mean + if err2 > 0: + err2 = 0 + Yerr[i][0].append(abs(err2)) + Yerr[i][1].append(err1) + else: + for i in range(0, numimpls): + Y[i].append(float(length) / matches[2 + i]) + i = 0 + for impl in impls: + colour = colours.next() + pylab.plot(X, Y[i], c=colour) + plot_done = True + # If we have error values then draw the bars + if len(Yerr[i]) > 0: + pylab.errorbar(X, Y[i], yerr=Yerr[i], c=colour, label=impl, + fmt='o') + else: + pylab.scatter(X, Y[i], c=colour, label=impl, + edgecolors='none') + i += 1 + # If we didn't draw anything then skip this graph + if not plot_done: + continue + pylab.legend(loc='upper left', ncol=3, prop={'size': 'small'}) + pylab.grid() + # Tidy up alignment text for benchmarks that need it + alignment = alignment.replace(" ", "") + alignment = alignment.replace("/", "-") + pylab.title('%s of %s byte aligned buffers' % (benchmark, alignment)) + pylab.xlabel('Length (B)') + pylab.ylabel('Bytes per cycle or ns') + + top = max(X) + + power = int(round(math.log(top) / math.log(2))) + + pylab.semilogx() + + pylab.axes().set_xticks([2**x for x in range(0, power+1)]) + pylab.axes().set_xticklabels([pretty_kb(2**x) + for x in range(0, power+1)]) + pylab.xlim(0, top) + pylab.ylim(0, pylab.ylim()[1]) + pylab.savefig(os.path.join(output_path, '%s-%s.png' % + (benchmark, alignment)), dpi=72) + +plotters = { + "memcpy" : plot_benchmark, + "strlen" : plot_benchmark, +} + +def plot_benchmark(directory, benchmark): + plotter = plotters.get(benchmark, plot_null) + input_path = os.path.join(directory, "bench-%s.*.out" % benchmark) + input_files = glob.glob(input_path) + if not input_files: + return + oldinput_path = os.path.join(directory, "bench-%s.*.out.old" % benchmark) + oldinput_files = glob.glob(oldinput_path) + plotter(benchmark, input_files, oldinput_files, directory) + +def usage(): + print "bench-plot.py <directory> <benchmark>" + sys.exit(1) + +if __name__ == '__main__': + if len(sys.argv) != 3: + usage() + directory = sys.argv[1] + benchmark = sys.argv[2] + plot_benchmark(directory, benchmark)