#!/usr/bin/env python """ parse bench.log file and produce ReST tables. positional arguments specify names of benchmark types to produce reports for. """ # XXX unify benchtype handling form runbench.py and report.py import py, os import smaps, runbench parser = py.compat.optparse.OptionParser(usage=__doc__) parser.add_option("-l", "--benchlog", action="store", dest="benchlog", default="bench.log", help="logfile to read results from") parser.add_option("-g", "--store-graphs", action="store", dest="basepath", default=None, help="optional path to store picture output") parser.add_option("-t", "--text", action="store_true", dest="text", help="run text reporting") parser.add_option("-r", "--no-rest", action="store_true", dest="norest", help="disable writing of ReST files"), parser.add_option("-s", "--show", action="store_true", dest="show", help="show pictures") def getmax(table): colmax = [] for row in table: if not colmax: colmax = [len(str(x)) for x in row] else: colmax = [max(len(str(x)),y) for x,y in zip(row + [0] * (len(colmax) - len(row)), colmax + [0] * (len(row) - len(colmax)))] return colmax def asciitable(table): colmax = getmax(table) lines = [] for row in table: line = [] for col,width in zip(row, colmax): line.append("%%-%ds" %(width) %(col)) lines.append(" ".join(line)) return "\n".join(lines) def ReSTtable(title, table): colmax = getmax(table) lines = [] l = ["-" * width for width in colmax] l.append("") l.insert(0, "") sepline = "+".join(l) lines.append(sepline) for row in table: if row: line = [""] for col,width in zip(row, colmax): line.append("%%-%ds" %(width) %(col)) line.append("") lines.append("|".join(line)) lines.append(sepline) else: lines.append(sepline) linesep = "\n " start = ".. table:: %s\n" %(title,) return start + linesep + linesep.join(lines) def maxtable_overview(resultset): tw = py.io.TerminalWriter() for name, results in resultset.getname2results(): tw.sep("=", name) row0 = "executable maxprivate maxrss".split() rows = [row0] for result in results: rows.append([result.executable, result.max("private"), result.max("rss"), ]) tw.line(asciitable(rows)) class ReporterBase(object): # subclasses need to have benchtype attribute def __init__(self, resultset): self.resultset = resultset.filter(benchtype=self.benchtype) def getexecutables(self, unique=True, short=True, pythonfirst=True): executables = [] for result in self.resultset.results: if short: name = result.executable_short else: name = str(result.executable) if not unique or name not in executables: if pythonfirst and name.startswith("python"): executables.insert(0, name) pythonfirst = False else: executables.append(name) return executables class CheckpointDetails(ReporterBase): benchtype = "objsizes" begin = report = end = lambda x: None # hooks def run_text(self): self.begin() for name, results in self.resultset.getname2results(): row0 = ["num"] + [result.executable for result in results] numsnapshots = min([len(i.snapshots) for i in results]) rows = [row0] for i in range(numsnapshots): row = [i] for result in results: row.append(result.snapshots[i].private) rows.append(row) self.tw.sep("=", "private RSS at checkpoints: %s" %(name,)) self.tw.line(asciitable(rows)) self.end() class IncrementalSizePerBench(ReporterBase): benchtype = "objsizes" def run_text(self): tw = py.io.TerminalWriter() tw.sep("=", "Incremental private RSS of size benchmarks") table = self.gettable() tw.line(asciitable(table)) def run_rest(self, filename="table-objsize.txt"): rows = self.gettable() p = py.path.local(filename) p.write(ReSTtable("Memory usage of single Python objects",rows)) print "wrote", p def gettable(self): executables = self.getexecutables(unique=True, short=True) row0 = ["name"] + [str(x) for x in executables] rows = [row0] name2results = self.resultset.getname2results() name2results.sort() # we rely on result ordering below, i.e. # each benchmark is run and recorded with the same interpreter ordering for name, results in name2results: basesize = self.get_incremental_size(results[0]) row = [name] for executable, result in zip(executables, results): assert result.executable_short == executable, "ordering assumtion" incsize = self.get_incremental_size(result) # colors work but messup table layout color = incsize <= basesize and "green" or "red" #row.append(tw.markup(str(incsize), **{color:True})) if basesize and incsize != basesize: row.append("**%.2f** / %d" %(incsize / (basesize*1.0), incsize)) else: row.append(incsize) rows.append(row) return rows def run_graphic(self, plotter): """ This function creates series of graphs for showing incremental private memory consumed for object allocation (without base interpreter size) """ resultset = self.resultset if not resultset.results: return allresults = resultset.getname2results() allresults.sort() names = [name for name, _ in allresults] benchresults = [[] for _ in range(len(allresults[0][1]))] executables = [result.executable for result in allresults[0][1]] for _, results in allresults: for i, result in enumerate(results): benchresults[i].append(self.get_incremental_size(result)) plotter.plot_objsizes(benchresults, names, executables) def get_incremental_size(self, result): # checkpoint 0: state before benchmark # checkpoint 1: state at end of benchmark func # checkpoint 2: state after benchmark func finished basesize = result.snapshots[0].heap_private() inbench = result.snapshots[1].heap_private() return inbench - basesize class BaseSizeOfInterpreters(ReporterBase): benchtype = "basesize" HEAP = runbench.Mappings.HEAP CODE = runbench.Mappings.CODE DATA = runbench.Mappings.DATA def getexecutables(self): return [r.executable for r in self.resultset.results] def run_text(self): tw = py.io.TerminalWriter() tw.sep("=", "Basesize of interpreters") executables = self.getexecutables() row0 = ['Group-Pagestate'] + executables rows = [row0] row_kinds = "shared_clean shared_dirty private_clean private_dirty".split() results = self.resultset.results def makerow(rowname, mapping_func): row = [rowname] for result in results: row.append(mapping_func(result)) rows.append(row) makerow("HEAP-RSS", lambda result: result.snapshot.filter(group=self.HEAP).rss) makerow("HEAP-private_clean", lambda result: result.snapshot.filter(group=self.HEAP).private_clean) makerow("HEAP-private_dirty", lambda result: result.snapshot.filter(group=self.HEAP).private_dirty) # we only show the clean bits of the code, this might not really work # out if the code is not compiled position-indepently, likewise the often # seen dirty code page might be a dispatch table (correct term?) of the linker makerow("IP-CODE-RSS", lambda result: # can be used to see if the linker dirtied the code segment result.snapshot.filter(group=result.executable, kind=self.CODE).rss) makerow("IP-CODE-shared_clean", lambda result: result.snapshot.filter(group=result.executable, kind=self.CODE).shared_clean) makerow("IP-CODE-private_clean", lambda result: result.snapshot.filter(group=result.executable, kind=self.CODE).private_clean) # whole static data of the process in memory, also including e.g. shared data with other processes makerow("IP-DATA-RSS", lambda result: result.snapshot.filter(group=result.executable, kind=self.DATA).rss) # static data that is not shared with another process and was not modified by the process # can be easily shared with another process makerow("IP-DATA-private_clean", lambda result: result.snapshot.filter(group=result.executable, kind=self.DATA).private_clean) # static data that is not shared with another process and was modified by the process makerow("IP-DATA-private_dirty", lambda result: result.snapshot.filter(group=result.executable, kind=self.DATA).private_dirty) # rest includes other shared libraries that are neither the interpreter nor the heap makerow("REST-private_clean", lambda result: result.snapshot.filter(group=result.executable, inv=True). filter(group=self.HEAP, inv=True).private_clean) makerow("REST-private_dirty", lambda result: result.snapshot.filter(group=result.executable, inv=True). filter(group=self.HEAP, inv=True).private_dirty) makerow("REST-RSS", lambda result: result.snapshot.filter(group=result.executable, inv=True). filter(group=self.HEAP, inv=True).rss) tw.line(asciitable(rows)) def run_rest(self, filename="table-basesize.txt"): p = py.path.local(filename) executables = self.getexecutables() row0 = ['Group-Pagestate'] + executables rows = [row0] row_kinds = "shared_clean shared_dirty private_clean private_dirty".split() results = self.resultset.results def makerow(rowname, mapping_func): row = [rowname] for result in results: row.append(mapping_func(result)) rows.append(row) #makerow("HEAP-RSS", lambda result: # result.snapshot.filter(group=self.HEAP).rss) #makerow("HEAP-shared_clean", lambda result: # result.snapshot.filter(group=self.HEAP).shared_clean) #makerow("HEAP-private_clean", lambda result: # result.snapshot.filter(group=self.HEAP).private_clean) makerow("HEAP private", lambda result: result.snapshot.filter(group=self.HEAP).private_dirty) # we only show the clean bits of the code, this might not really work # out if the code is not compiled position-indepently, likewise the often # seen dirty code page might be a dispatch table (correct term?) of the linker makerow("IP-CODE share", lambda result: # can be used to see if the linker dirtied the code segment result.snapshot.filter(group=result.executable, kind=self.CODE).rss) #makerow("IP-CODE-shared_clean", lambda result: # result.snapshot.filter(group=result.executable, kind=self.CODE).shared_clean) #makerow("IP-CODE-private_clean", lambda result: # result.snapshot.filter(group=result.executable, kind=self.CODE).private_clean) # whole static data of the process in memory, also including e.g. shared data with other processes #makerow("IP-DATA-RSS", lambda result: # result.snapshot.filter(group=result.executable, kind=self.DATA).rss) # static data that is not shared with another process and was not modified by the process # can be easily shared with another process makerow("IP-DATA sharable", lambda result: result.snapshot.filter(group=result.executable, kind=self.DATA).shared_clean + result.snapshot.filter(group=result.executable, kind=self.DATA).private_clean) #makerow("IP-DATA-private_clean", lambda result: # result.snapshot.filter(group=result.executable, kind=self.DATA).private_clean) # static data that is not shared with another process and was modified by the process makerow("IP-DATA private", lambda result: result.snapshot.filter(group=result.executable, kind=self.DATA).private_dirty) # rest includes other shared libraries that are neither the interpreter nor the heap #makerow("REST-private_clean", lambda result: # result.snapshot.filter(group=result.executable, inv=True). # filter(group=self.HEAP, inv=True).private_clean) makerow("REST private", lambda result: result.snapshot.filter(group=result.executable, inv=True). filter(group=self.HEAP, inv=True).private_dirty) makerow("REST shared ", lambda result: result.snapshot.filter(group=result.executable, inv=True) .filter(group=self.HEAP, inv=True).private_clean + result.snapshot.filter(group=result.executable, inv=True) .filter(group=self.HEAP, inv=True).shared_clean) p.write(ReSTtable("Interpreter resident base usage", rows)) print "writing", p def run_graphic(self, plotter): """ This function plots base interpreter sizes of various interpreters with bars specifying: * heap private * ip code * ip data * private rest """ results = self.resultset.results if not results: return import numpy heap_private = numpy.array([result.snapshot.filter(group=self.HEAP).private for result in results]) ip_code = numpy.array([result.snapshot.filter(group=result.executable, kind=self.CODE). private for result in results]) ip_data = numpy.array([result.snapshot.filter(group=result.executable, kind=self.DATA). private for result in results]) rest = numpy.array([result.snapshot.filter(group=result.executable, inv=True). filter(group=self.HEAP, inv=True).private for result in results]) plotter.plot_baseint_sizes(heap_private, ip_code, ip_data, rest, [r.executable for r in results]) return dict([(r.executable, r.snapshot) for r in results]) class BaseTimeOfInterpreters(ReporterBase): benchtype = "basetime" def run_text(self): tw = py.io.TerminalWriter() tw.sep("=", "Basetime of interpreters") # result.mintimings -> [(name, timings_dict)] executables = self.getexecutables(short=True) row0 = "run real user sys".split() rows = [row0] names = [x[0] for x in self.resultset.results[0].mintimings] for selectname in names: for result in self.resultset.results: for name, timing in result.mintimings: if name == selectname: rows.append(["%s-%s" %(result.executable, name), timing['real'], timing['user'], timing['sys']]) rows.append([]) tw.line(asciitable(rows)) def run_rest(self, filename="table-basetime.txt"): p = py.path.local(filename) executables = self.getexecutables(pythonfirst=False, short=True) row0 = ["startup"] + executables rows = [row0] # result.mintimings -> [(name, timings_dict)] names = [x[0] for x in self.resultset.results[0].mintimings] for startup in names: row = [startup] for result in self.resultset.results: for name, timing in result.mintimings: if name == startup: row.append("**%0.2f**/%.02f" % (timing['real'], timing['user'])) #, timing['sys'])) rows.append(row) p.write(ReSTtable("Interpreter startup time", rows)) print "wrote", p class BenchTimeOfInterpreters(ReporterBase): benchtype = "benchtime" def run_text(self): tw = py.io.TerminalWriter() tw.sep("=", "Timed benchmarks") rows = self.generate_table() tw.line(asciitable(rows)) def run_rest(self, filename="table-benchtime.txt"): p = py.path.local(filename) rows = self.generate_table() p.write(ReSTtable("Popular speed benchmarks", rows)) print "wrote", p def generate_table(self): # result.mintimings -> [(name, timings_dict)] # order by name benchname2executable2mintimings = {} for result in self.resultset.results: for name,mintimings in result.mintimings: d = benchname2executable2mintimings.setdefault(name, {}) assert result.executable_short not in d d[result.executable_short] = mintimings benchnames = benchname2executable2mintimings.keys() benchnames.sort() executables = self.getexecutables(pythonfirst=True) row0 = ["name"] + executables rows = [row0] for name in benchnames: row = [name] reference = None for executable in row0[1:]: timing = benchname2executable2mintimings[name][executable] perf = timing['perf'] if reference is None: reference = perf cell = "**%.2f** / %d" % (perf / reference, perf) row.append(cell) rows.append(row) return rows class Appprofiles(ReporterBase): benchtype = "appprofiles" def __init__(self, resultset): self.full_resultset = resultset super(Appprofiles, self).__init__(resultset) def run_text(self): tw = py.io.TerminalWriter() tw.sep("=", "Appprofiles memory sampling") rows = self.generate_large_table() tw.line(asciitable(rows)) def run_rest(self, filename="table-appprofiles.txt"): p = py.path.local(filename) rows = self.generate_table() p.write(ReSTtable("Application benchmarhs: Maximum and Average interpreter data", rows)) print "wrote", p def run_graphic(self, plotter): # XXX get rid of this ugly dependency snapshots = BaseSizeOfInterpreters(self.full_resultset).run_graphic() name2result = self.resultset.getname2results() plotter.plot_appprofiles(name2result, snapshots) plotter.plot_appprofiles(name2result, totals) def generate_table(self): executables = self.getexecutables(unique=True, short=True) row0 = ["benchmark"] + [str(x) for x in executables] rows = [row0] name2results = self.resultset.getname2results() name2results.sort() for name, results in name2results: row = [name] for executable, result in zip(executables, results): assert result.executable_short == executable, "ordering assumtion" row.append("**%s** / %d" % (result.max, result.avg)) rows.append(row) return rows def generate_large_table(self): executables = self.getexecutables(short=True) row0 = "interpreter benchmark duration #snapshots heapdata_min heapdata_avg heapdata_max" row0 += "dirtyd_min dirtyd_avg dirtyd_max min_data avg_data max_data code_min code_avg code_max" row0 = row0.split() rows = [row0] names = [] for result in self.resultset.results: if result.benchname not in names: names.append(result.benchname) stats = {} attrs = ["min", "avg", "max"] attrs.extend([ "min_dirtied_data", "avg_dirtied_data", "max_dirtied_data", "min_data", "avg_data", "max_data", "min_code", "avg_code", "max_code", ]) for name in names: for result in self.resultset.results: if name == result.benchname: timestamps = [float(ss.timestamp) for ss in result.snapshots] min_ts, max_ts = min(timestamps), max(timestamps) row = [result.executable_short, name, "%.2f" % (max_ts - min_ts, ), len(result.snapshots)] for attr in attrs: for kind, default in ((min, 2**31), (max, 0)): key = (result.executable_short, kind.__name__) valuedict = stats.setdefault(key, {}) old = valuedict.get(attr, default) data = getattr(result, attr) valuedict[attr] = kind(old, data) data_str = str(data) if "avg" in attr: data_str = "%.2f" % data row.append(data_str) rows.append(row) # calculate min/max of every (interpreter, column) pair for executable in executables: for kind in (min, max): kindname = kind.__name__ key = (executable, kind.__name__) row = [executable, kindname, "-", "-"] for attr in attrs: data = stats[key][attr] data_str = str(data) if "avg" in attr: data_str = "%.2f" % data row.append(data_str) rows.append(row) return rows class Pauses(ReporterBase): benchtype = "pauses" def run_rest(self, filename="table-pauses.txt"): p = py.path.local(filename) rows = self.generate_table() p.write(ReSTtable("bytecode execution pauses", rows)) print "wrote", p def run_text(self): tw = py.io.TerminalWriter() tw.sep("=", "Pauses between every bytecode") rows = self.generate_table() tw.line(asciitable(rows)) def generate_table(self): executables = self.getexecutables() row0 = ["bench"] + executables rows = [row0] # order by benchname benchname2executable2result = {} for result in self.resultset.results: d = benchname2executable2result.setdefault(result.benchname, {}) assert result.executable_short not in d d[result.executable_short] = result benchnames = benchname2executable2result.keys() benchnames.sort() for benchname in benchnames: row = [benchname] for executable in row0[1:]: result = benchname2executable2result[benchname][executable] maxpause = max(result.lst) row.append("%.2fs" %(maxpause,)) rows.append(row) return rows def run_graphic(self, plotter): plotter.plot_pausehistogram(self.resultset) if __name__ == "__main__": options, args = parser.parse_args() type2reporter = {} for X in globals().values(): benchtype = getattr(X, 'benchtype', None) if benchtype is not None: type2reporter.setdefault(benchtype, []).append(X) if not args: args = type2reporter.keys() args.sort() for arg in args: assert arg in type2reporter, "unknown benchtype: %r" %(arg,) benchlog = py.path.local(options.benchlog) resultset = runbench.ResultSet() print "parsing logfile", benchlog resultset.parse(benchlog) for arg in args: for cls in type2reporter[arg]: reporter = cls(resultset) if not reporter.resultset: print "skipping %s reporting, no results found" %(cls.__name__, ) continue if options.text and hasattr(reporter, 'run_text'): reporter.run_text() if options.basepath or options.show: if hasattr(reporter, 'run_graphic'): from report_graphic import Plotter plotter = Plotter(options.basepath, options.show) reporter.run_graphic(plotter) if not options.norest: if hasattr(reporter, 'run_rest'): reporter.run_rest()