[RFC] benchtests/bench-plot.py: Add graphing script for string benchmarks.

Message ID 52274894.4000604@linaro.org
State Rejected
Headers show

Commit Message

Will Newton Sept. 4, 2013, 2:49 p.m.
Add a Python script that uses pylab to graph the results of string
benchmarks. Currently it only supports graphing the results of the
strlen and memcpy benchmarks.

ChangeLog:

2013-09-03   Will Newton  <will.newton@linaro.org>

	* benchtests/bench-plot.py: New file.
---
 benchtests/bench-plot.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 184 insertions(+)
 create mode 100755 benchtests/bench-plot.py

Comments

Siddhesh Poyarekar Dec. 6, 2013, 8:22 a.m. | #1
On Wed, Sep 04, 2013 at 03:49:56PM +0100, Will Newton wrote:
> 
> Add a Python script that uses pylab to graph the results of string
> benchmarks. Currently it only supports graphing the results of the
> strlen and memcpy benchmarks.

Sorry Will, I had this marked for review and I completely forgot about
it.  The script doesn't seem to work with the current files, probably
because the -ifunc tests were merged in.  Would you be able to post an
updated version?

I am OK with having a python script to do this; in fact I worked on
porting my benchmark parsing script (which I'll post soon) since I
found it easier to write a more readable program compared to perl.
However, I believe there were reservations in the past about
introducing additional dependencies, so I'm wondering if we could make
an exception here and add a dependency on python for running
benchmarks.  I'll start a discussion on that in a separate thread when
I post my ported script.

Thanks,
Siddhesh

> 
> ChangeLog:
> 
> 2013-09-03   Will Newton  <will.newton@linaro.org>
> 
> 	* benchtests/bench-plot.py: New file.
> ---
>  benchtests/bench-plot.py | 184 +++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 184 insertions(+)
>  create mode 100755 benchtests/bench-plot.py
> 
> diff --git a/benchtests/bench-plot.py b/benchtests/bench-plot.py
> new file mode 100755
> index 0000000..7fd680b
> --- /dev/null
> +++ b/benchtests/bench-plot.py
> @@ -0,0 +1,184 @@
> +#!/usr/bin/env python
> +# Copyright (C) 2013 Free Software Foundation, Inc.
> +# This file is part of the GNU C Library.
> +
> +# The GNU C Library is free software; you can redistribute it and/or
> +# modify it under the terms of the GNU Lesser General Public
> +# License as published by the Free Software Foundation; either
> +# version 2.1 of the License, or (at your option) any later version.
> +
> +# The GNU C Library is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +# Lesser General Public License for more details.
> +
> +# You should have received a copy of the GNU Lesser General Public
> +# License along with the GNU C Library; if not, see
> +# <http://www.gnu.org/licenses/>.
> +
> +# A script for graphing the results of glibc string benchmarks.
> +
> +import glob
> +import math
> +import os
> +import re
> +import sys
> +
> +import pylab
> +
> +def unique_values(rows, column):
> +    values = []
> +    for row in rows:
> +        if not row[column] in values:
> +            values.append(row[column])
> +    return sorted(values)
> +
> +def make_colours():
> +    return iter('m b g r c y k pink orange brown grey'.split())
> +
> +def pretty_kb(v):
> +    if v < 1024:
> +        return '%d' % v
> +    else:
> +        if v % 1024 == 0:
> +            return '%d k' % (v//1024)
> +        else:
> +            return '%.1f k' % (v/1024)
> +
> +def plot_null(benchmark, input_files, oldinput_files, output_path):
> +    pass
> +
> +def parse_benchmark_rows(input_files):
> +    impls = []
> +    rows = []
> +    matcher = re.compile("Length\s+(\d+), alignment\s+([0-9 /]+):")
> +    for input_file in input_files:
> +        lines = open(input_file).readlines()
> +        if not impls:
> +            impls = lines[0].strip().split("\t")
> +        for line in lines[1:]:
> +            columns = line.split("\t")
> +            m = matcher.match(columns[0])
> +            groups = m.groups()
> +            length = groups[0]
> +            alignment = groups[1]
> +            rows.append([int(length), alignment] + map(float, columns[1:]))
> +    return (impls, rows)
> +
> +def plot_benchmark(benchmark, input_files, oldinput_files, output_path):
> +    (newimpls, newrows) = parse_benchmark_rows(input_files)
> +    (oldimpls, oldrows) = parse_benchmark_rows(oldinput_files)
> +    # We're only interested in the glibc impl for the old files
> +    if oldimpls:
> +        oldimpls = ["%s (old)" % oldimpls[0]]
> +    alignments = unique_values(newrows, 1)
> +    for alignment in alignments:
> +        # Draw one figure per alignment value
> +        pylab.figure(1).set_size_inches((12, 10))
> +        pylab.clf()
> +        plot_done = False
> +        colours = make_colours()
> +        for (impls, rows) in ((newimpls, newrows), (oldimpls, oldrows)):
> +            if not rows:
> +                continue
> +            plot_rows = []
> +            for row in rows:
> +                if row[1] == alignment:
> +                    plot_rows.append(row)
> +            X = unique_values(plot_rows, 0)
> +            # Filter out zero length entries
> +            X = [x for x in X if x > 0]
> +            # If there are too few data points, skip this alignment
> +            if len(X) < 2:
> +                continue
> +            numimpls = len(impls)
> +            Y = []
> +            Yerr = []
> +            # Initialize the Y and Yerr arrays
> +            for i in range(0, numimpls):
> +                Y.append([])
> +                Yerr.append([[], []])
> +            for length in X:
> +                matches = [x for x in plot_rows if x[0] == length]
> +                # If there is more than one test run for a given test
> +                # then calculate the mean and min/max
> +                if len(matches) > 1:
> +                    for i in range(0, numimpls):
> +                        vals = [x[2 + i] for x in matches]
> +                        mean = length / (sum(vals)/len(vals))
> +                        Y[i].append(mean)
> +                        err1 = (length / max(vals)) - mean
> +                        if err1 < 0:
> +                            err1 = 0
> +                        err2 = (length / min(vals)) - mean
> +                        if err2 > 0:
> +                            err2 = 0
> +                        Yerr[i][0].append(abs(err2))
> +                        Yerr[i][1].append(err1)
> +                else:
> +                    for i in range(0, numimpls):
> +                        Y[i].append(float(length) / matches[2 + i])
> +            i = 0
> +            for impl in impls:
> +                colour = colours.next()
> +                pylab.plot(X, Y[i], c=colour)
> +                plot_done = True
> +                # If we have error values then draw the bars
> +                if len(Yerr[i]) > 0:
> +                    pylab.errorbar(X, Y[i], yerr=Yerr[i], c=colour, label=impl,
> +                                   fmt='o')
> +                else:
> +                    pylab.scatter(X, Y[i], c=colour, label=impl,
> +                                  edgecolors='none')
> +                i += 1
> +        # If we didn't draw anything then skip this graph
> +        if not plot_done:
> +            continue
> +        pylab.legend(loc='upper left', ncol=3, prop={'size': 'small'})
> +        pylab.grid()
> +        # Tidy up alignment text for benchmarks that need it
> +        alignment = alignment.replace(" ", "")
> +        alignment = alignment.replace("/", "-")
> +        pylab.title('%s of %s byte aligned buffers' % (benchmark, alignment))
> +        pylab.xlabel('Length (B)')
> +        pylab.ylabel('Bytes per cycle or ns')
> +
> +        top = max(X)
> +
> +        power = int(round(math.log(top) / math.log(2)))
> +
> +        pylab.semilogx()
> +
> +        pylab.axes().set_xticks([2**x for x in range(0, power+1)])
> +        pylab.axes().set_xticklabels([pretty_kb(2**x)
> +                                      for x in range(0, power+1)])
> +        pylab.xlim(0, top)
> +        pylab.ylim(0, pylab.ylim()[1])
> +        pylab.savefig(os.path.join(output_path, '%s-%s.png' %
> +                                   (benchmark, alignment)), dpi=72)
> +
> +plotters = {
> +    "memcpy" : plot_benchmark,
> +    "strlen" : plot_benchmark,
> +}
> +
> +def plot_benchmark(directory, benchmark):
> +    plotter = plotters.get(benchmark, plot_null)
> +    input_path = os.path.join(directory, "bench-%s.*.out" % benchmark)
> +    input_files = glob.glob(input_path)
> +    if not input_files:
> +        return
> +    oldinput_path = os.path.join(directory, "bench-%s.*.out.old" % benchmark)
> +    oldinput_files = glob.glob(oldinput_path)
> +    plotter(benchmark, input_files, oldinput_files, directory)
> +
> +def usage():
> +    print "bench-plot.py <directory> <benchmark>"
> +    sys.exit(1)
> +
> +if __name__ == '__main__':
> +    if len(sys.argv) != 3:
> +        usage()
> +    directory = sys.argv[1]
> +    benchmark = sys.argv[2]
> +    plot_benchmark(directory, benchmark)
> -- 
> 1.8.1.4
>

Patch

diff --git a/benchtests/bench-plot.py b/benchtests/bench-plot.py
new file mode 100755
index 0000000..7fd680b
--- /dev/null
+++ b/benchtests/bench-plot.py
@@ -0,0 +1,184 @@ 
+#!/usr/bin/env python
+# Copyright (C) 2013 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+# A script for graphing the results of glibc string benchmarks.
+
+import glob
+import math
+import os
+import re
+import sys
+
+import pylab
+
+def unique_values(rows, column):
+    values = []
+    for row in rows:
+        if not row[column] in values:
+            values.append(row[column])
+    return sorted(values)
+
+def make_colours():
+    return iter('m b g r c y k pink orange brown grey'.split())
+
+def pretty_kb(v):
+    if v < 1024:
+        return '%d' % v
+    else:
+        if v % 1024 == 0:
+            return '%d k' % (v//1024)
+        else:
+            return '%.1f k' % (v/1024)
+
+def plot_null(benchmark, input_files, oldinput_files, output_path):
+    pass
+
+def parse_benchmark_rows(input_files):
+    impls = []
+    rows = []
+    matcher = re.compile("Length\s+(\d+), alignment\s+([0-9 /]+):")
+    for input_file in input_files:
+        lines = open(input_file).readlines()
+        if not impls:
+            impls = lines[0].strip().split("\t")
+        for line in lines[1:]:
+            columns = line.split("\t")
+            m = matcher.match(columns[0])
+            groups = m.groups()
+            length = groups[0]
+            alignment = groups[1]
+            rows.append([int(length), alignment] + map(float, columns[1:]))
+    return (impls, rows)
+
+def plot_benchmark(benchmark, input_files, oldinput_files, output_path):
+    (newimpls, newrows) = parse_benchmark_rows(input_files)
+    (oldimpls, oldrows) = parse_benchmark_rows(oldinput_files)
+    # We're only interested in the glibc impl for the old files
+    if oldimpls:
+        oldimpls = ["%s (old)" % oldimpls[0]]
+    alignments = unique_values(newrows, 1)
+    for alignment in alignments:
+        # Draw one figure per alignment value
+        pylab.figure(1).set_size_inches((12, 10))
+        pylab.clf()
+        plot_done = False
+        colours = make_colours()
+        for (impls, rows) in ((newimpls, newrows), (oldimpls, oldrows)):
+            if not rows:
+                continue
+            plot_rows = []
+            for row in rows:
+                if row[1] == alignment:
+                    plot_rows.append(row)
+            X = unique_values(plot_rows, 0)
+            # Filter out zero length entries
+            X = [x for x in X if x > 0]
+            # If there are too few data points, skip this alignment
+            if len(X) < 2:
+                continue
+            numimpls = len(impls)
+            Y = []
+            Yerr = []
+            # Initialize the Y and Yerr arrays
+            for i in range(0, numimpls):
+                Y.append([])
+                Yerr.append([[], []])
+            for length in X:
+                matches = [x for x in plot_rows if x[0] == length]
+                # If there is more than one test run for a given test
+                # then calculate the mean and min/max
+                if len(matches) > 1:
+                    for i in range(0, numimpls):
+                        vals = [x[2 + i] for x in matches]
+                        mean = length / (sum(vals)/len(vals))
+                        Y[i].append(mean)
+                        err1 = (length / max(vals)) - mean
+                        if err1 < 0:
+                            err1 = 0
+                        err2 = (length / min(vals)) - mean
+                        if err2 > 0:
+                            err2 = 0
+                        Yerr[i][0].append(abs(err2))
+                        Yerr[i][1].append(err1)
+                else:
+                    for i in range(0, numimpls):
+                        Y[i].append(float(length) / matches[2 + i])
+            i = 0
+            for impl in impls:
+                colour = colours.next()
+                pylab.plot(X, Y[i], c=colour)
+                plot_done = True
+                # If we have error values then draw the bars
+                if len(Yerr[i]) > 0:
+                    pylab.errorbar(X, Y[i], yerr=Yerr[i], c=colour, label=impl,
+                                   fmt='o')
+                else:
+                    pylab.scatter(X, Y[i], c=colour, label=impl,
+                                  edgecolors='none')
+                i += 1
+        # If we didn't draw anything then skip this graph
+        if not plot_done:
+            continue
+        pylab.legend(loc='upper left', ncol=3, prop={'size': 'small'})
+        pylab.grid()
+        # Tidy up alignment text for benchmarks that need it
+        alignment = alignment.replace(" ", "")
+        alignment = alignment.replace("/", "-")
+        pylab.title('%s of %s byte aligned buffers' % (benchmark, alignment))
+        pylab.xlabel('Length (B)')
+        pylab.ylabel('Bytes per cycle or ns')
+
+        top = max(X)
+
+        power = int(round(math.log(top) / math.log(2)))
+
+        pylab.semilogx()
+
+        pylab.axes().set_xticks([2**x for x in range(0, power+1)])
+        pylab.axes().set_xticklabels([pretty_kb(2**x)
+                                      for x in range(0, power+1)])
+        pylab.xlim(0, top)
+        pylab.ylim(0, pylab.ylim()[1])
+        pylab.savefig(os.path.join(output_path, '%s-%s.png' %
+                                   (benchmark, alignment)), dpi=72)
+
+plotters = {
+    "memcpy" : plot_benchmark,
+    "strlen" : plot_benchmark,
+}
+
+def plot_benchmark(directory, benchmark):
+    plotter = plotters.get(benchmark, plot_null)
+    input_path = os.path.join(directory, "bench-%s.*.out" % benchmark)
+    input_files = glob.glob(input_path)
+    if not input_files:
+        return
+    oldinput_path = os.path.join(directory, "bench-%s.*.out.old" % benchmark)
+    oldinput_files = glob.glob(oldinput_path)
+    plotter(benchmark, input_files, oldinput_files, directory)
+
+def usage():
+    print "bench-plot.py <directory> <benchmark>"
+    sys.exit(1)
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        usage()
+    directory = sys.argv[1]
+    benchmark = sys.argv[2]
+    plot_benchmark(directory, benchmark)