poky/scripts/oe-build-perf-report

#!/usr/bin/python3

Examine build performance test results

Copyright (c) 2017, Intel Corporation.

This program is free software; you can redistribute it and/or modify it

under the terms and conditions of the GNU General Public License,

version 2, as published by the Free Software Foundation.

This program is distributed in the hope it will be useful, but WITHOUT

ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for

more details.

import argparse import json import logging import os import re import sys from collections import namedtuple, OrderedDict from operator import attrgetter from xml.etree import ElementTree as ET

Import oe libs

scripts_path = os.path.dirname(os.path.realpath(file)) sys.path.append(os.path.join(scripts_path, 'lib')) import scriptpath from build_perf import print_table from build_perf.report import (metadata_xml_to_json, results_xml_to_json, aggregate_data, aggregate_metadata, measurement_stats, AggregateTestData) from build_perf import html from buildstats import BuildStats, diff_buildstats, BSVerDiff

scriptpath.add_oe_lib_path()

from oeqa.utils.git import GitRepo, GitError

Setup logging

logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") log = logging.getLogger('oe-build-perf-report')

Container class for tester revisions

TestedRev = namedtuple('TestedRev', 'commit commit_number tags')

def get_test_runs(repo, tag_name, **kwargs): """Get a sorted list of test runs, matching given pattern""" # First, get field names from the tag name pattern field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)] undef_fields = [f for f in field_names if f not in kwargs.keys()]

# Fields for formatting tag name pattern
str_fields = dict([(f, '*') for f in field_names])
str_fields.update(kwargs)

# Get a list of all matching tags
tag_pattern = tag_name.format(**str_fields)
tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)

# Parse undefined fields from tag names
str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
# escape parenthesis in fields in order to not messa up the regexp
fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
str_fields.update(fixed_fields)
tag_re = re.compile(tag_name.format(**str_fields))

# Parse fields from tags
revs = []
for tag in tags:
    m = tag_re.match(tag)
    groups = m.groupdict()
    revs.append([groups[f] for f in undef_fields] + [tag])

# Return field names and a sorted list of revs
return undef_fields, sorted(revs)

def list_test_revs(repo, tag_name, verbosity, **kwargs): """Get list of all tested revisions""" valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])

fields, revs = get_test_runs(repo, tag_name, **valid_kwargs)
ignore_fields = ['tag_number']
if verbosity < 2:
    extra_fields = ['COMMITS', 'TEST RUNS']
    ignore_fields.extend(['commit_number', 'commit'])
else:
    extra_fields = ['TEST RUNS']

print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields]

# Sort revs
rows = [[fields[i].upper() for i in print_fields] + extra_fields]

prev = [''] * len(print_fields)
prev_commit = None
commit_cnt = 0
commit_field = fields.index('commit')
for rev in revs:
    # Only use fields that we want to print
    cols = [rev[i] for i in print_fields]


    if cols != prev:
        commit_cnt = 1
        test_run_cnt = 1
        new_row = [''] * (len(print_fields) + len(extra_fields))

        for i in print_fields:
            if cols[i] != prev[i]:
                break
        new_row[i:-len(extra_fields)] = cols[i:]
        rows.append(new_row)
    else:
        if rev[commit_field] != prev_commit:
            commit_cnt += 1
        test_run_cnt += 1

    if verbosity < 2:
        new_row[-2] = commit_cnt
    new_row[-1] = test_run_cnt
    prev = cols
    prev_commit = rev[commit_field]

print_table(rows)

def get_test_revs(repo, tag_name, **kwargs): """Get list of all tested revisions""" fields, runs = get_test_runs(repo, tag_name, **kwargs)

revs = {}
commit_i = fields.index('commit')
commit_num_i = fields.index('commit_number')
for run in runs:
    commit = run[commit_i]
    commit_num = run[commit_num_i]
    tag = run[-1]
    if not commit in revs:
        revs[commit] = TestedRev(commit, commit_num, [tag])
    else:
        assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
        revs[commit].tags.append(tag)

# Return in sorted table
revs = sorted(revs.values(), key=attrgetter('commit_number'))
log.debug("Found %d tested revisions:\n    %s", len(revs),
          "\n    ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
return revs

def rev_find(revs, attr, val): """Search from a list of TestedRev""" for i, rev in enumerate(revs): if getattr(rev, attr) == val: return i raise ValueError("Unable to find '{}' value '{}'".format(attr, val))

def is_xml_format(repo, commit): """Check if the commit contains xml (or json) data""" if repo.rev_parse(commit + ':results.xml'): log.debug("Detected report in xml format in %s", commit) return True else: log.debug("No xml report in %s, assuming json formatted results", commit) return False

def read_results(repo, tags, xml=True): """Read result files from repo"""

def parse_xml_stream(data):
    """Parse multiple concatenated XML objects"""
    objs = []
    xml_d = ""
    for line in data.splitlines():
        if xml_d and line.startswith('<?xml version='):
            objs.append(ET.fromstring(xml_d))
            xml_d = line
        else:
            xml_d += line
    objs.append(ET.fromstring(xml_d))
    return objs

def parse_json_stream(data):
    """Parse multiple concatenated JSON objects"""
    objs = []
    json_d = ""
    for line in data.splitlines():
        if line == '}{':
            json_d += '}'
            objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
            json_d = '{'
        else:
            json_d += line
    objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
    return objs

num_revs = len(tags)

# Optimize by reading all data with one git command
log.debug("Loading raw result data from %d tags, %s...", num_revs, tags[0])
if xml:
    git_objs = [tag + ':metadata.xml' for tag in tags] + [tag + ':results.xml' for tag in tags]
    data = parse_xml_stream(repo.run_cmd(['show'] + git_objs + ['--']))
    return ([metadata_xml_to_json(e) for e in data[0:num_revs]],
            [results_xml_to_json(e) for e in data[num_revs:]])
else:
    git_objs = [tag + ':metadata.json' for tag in tags] + [tag + ':results.json' for tag in tags]
    data = parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--']))
    return data[0:num_revs], data[num_revs:]

def get_data_item(data, key): """Nested getitem lookup""" for k in key.split('.'): data = data[k] return data

def metadata_diff(metadata_l, metadata_r): """Prepare a metadata diff for printing""" keys = [('Hostname', 'hostname', 'hostname'), ('Branch', 'branch', 'layers.meta.branch'), ('Commit number', 'commit_num', 'layers.meta.commit_count'), ('Commit', 'commit', 'layers.meta.commit'), ('Number of test runs', 'testrun_count', 'testrun_count') ]

def _metadata_diff(key):
    """Diff metadata from two test reports"""
    try:
        val1 = get_data_item(metadata_l, key)
    except KeyError:
        val1 = '(N/A)'
    try:
        val2 = get_data_item(metadata_r, key)
    except KeyError:
        val2 = '(N/A)'
    return val1, val2

metadata = OrderedDict()
for title, key, key_json in keys:
    value_l, value_r = _metadata_diff(key_json)
    metadata[key] = {'title': title,
                     'value_old': value_l,
                     'value': value_r}
return metadata

def print_diff_report(metadata_l, data_l, metadata_r, data_r): """Print differences between two data sets"""

# First, print general metadata
print("\nTEST METADATA:\n==============")
meta_diff = metadata_diff(metadata_l, metadata_r)
rows = []
row_fmt = ['{:{wid}} ', '{:<{wid}}   ', '{:<{wid}}']
rows = [['', 'CURRENT COMMIT', 'COMPARING WITH']]
for key, val in meta_diff.items():
    # Shorten commit hashes
    if key == 'commit':
        rows.append([val['title'] + ':', val['value'][:20], val['value_old'][:20]])
    else:
        rows.append([val['title'] + ':', val['value'], val['value_old']])
print_table(rows, row_fmt)


# Print test results
print("\nTEST RESULTS:\n=============")

tests = list(data_l['tests'].keys())
# Append tests that are only present in 'right' set
tests += [t for t in list(data_r['tests'].keys()) if t not in tests]

# Prepare data to be printed
rows = []
row_fmt = ['{:8}', '{:{wid}}', '{:{wid}}', '  {:>{wid}}', ' {:{wid}} ', '{:{wid}}',
           '  {:>{wid}}', '  {:>{wid}}']
num_cols = len(row_fmt)
for test in tests:
    test_l = data_l['tests'][test] if test in data_l['tests'] else None
    test_r = data_r['tests'][test] if test in data_r['tests'] else None
    pref = ' '
    if test_l is None:
        pref = '+'
    elif test_r is None:
        pref = '-'
    descr = test_l['description'] if test_l else test_r['description']
    heading = "{} {}: {}".format(pref, test, descr)

    rows.append([heading])

    # Generate the list of measurements
    meas_l = test_l['measurements'] if test_l else {}
    meas_r = test_r['measurements'] if test_r else {}
    measurements = list(meas_l.keys())
    measurements += [m for m in list(meas_r.keys()) if m not in measurements]

    for meas in measurements:
        m_pref = ' '
        if meas in meas_l:
            stats_l = measurement_stats(meas_l[meas], 'l.')
        else:
            stats_l = measurement_stats(None, 'l.')
            m_pref = '+'
        if meas in meas_r:
            stats_r = measurement_stats(meas_r[meas], 'r.')
        else:
            stats_r = measurement_stats(None, 'r.')
            m_pref = '-'
        stats = stats_l.copy()
        stats.update(stats_r)

        absdiff = stats['val_cls'](stats['r.mean'] - stats['l.mean'])
        reldiff = "{:+.1f} %".format(absdiff * 100 / stats['l.mean'])
        if stats['r.mean'] > stats['l.mean']:
            absdiff = '+' + str(absdiff)
        else:
            absdiff = str(absdiff)
        rows.append(['', m_pref, stats['name'] + ' ' + stats['quantity'],
                     str(stats['l.mean']), '->', str(stats['r.mean']),
                     absdiff, reldiff])
    rows.append([''] * num_cols)

print_table(rows, row_fmt)

print()

class BSSummary(object): def init(self, bs1, bs2): self.tasks = {'count': bs2.num_tasks, 'change': '{:+d}'.format(bs2.num_tasks - bs1.num_tasks)} self.top_consumer = None self.top_decrease = None self.top_increase = None self.ver_diff = OrderedDict()

    tasks_diff = diff_buildstats(bs1, bs2, 'cputime')

    # Get top consumers of resources
    tasks_diff = sorted(tasks_diff, key=attrgetter('value2'))
    self.top_consumer = tasks_diff[-5:]

    # Get biggest increase and decrease in resource usage
    tasks_diff = sorted(tasks_diff, key=attrgetter('absdiff'))
    self.top_decrease = tasks_diff[0:5]
    self.top_increase = tasks_diff[-5:]

    # Compare recipe versions and prepare data for display
    ver_diff = BSVerDiff(bs1, bs2)
    if ver_diff:
        if ver_diff.new:
            self.ver_diff['New recipes'] = [(n, r.evr) for n, r in ver_diff.new.items()]
        if ver_diff.dropped:
            self.ver_diff['Dropped recipes'] = [(n, r.evr) for n, r in ver_diff.dropped.items()]
        if ver_diff.echanged:
            self.ver_diff['Epoch changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.echanged.items()]
        if ver_diff.vchanged:
            self.ver_diff['Version changed'] = [(n, "{} → {}".format(r.left.version, r.right.version)) for n, r in ver_diff.vchanged.items()]
        if ver_diff.rchanged:
            self.ver_diff['Revision changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.rchanged.items()]

def print_html_report(data, id_comp, buildstats): """Print report in html format""" # Handle metadata metadata = metadata_diff(data[id_comp].metadata, data[-1].metadata)

# Generate list of tests
tests = []
for test in data[-1].results['tests'].keys():
    test_r = data[-1].results['tests'][test]
    new_test = {'name': test_r['name'],
                'description': test_r['description'],
                'status': test_r['status'],
                'measurements': [],
                'err_type': test_r.get('err_type'),
               }
    # Limit length of err output shown
    if 'message' in test_r:
        lines = test_r['message'].splitlines()
        if len(lines) > 20:
            new_test['message'] = '...\n' + '\n'.join(lines[-20:])
        else:
            new_test['message'] = test_r['message']


    # Generate the list of measurements
    for meas in test_r['measurements'].keys():
        meas_r = test_r['measurements'][meas]
        meas_type = 'time' if meas_r['type'] == 'sysres' else 'size'
        new_meas = {'name': meas_r['name'],
                    'legend': meas_r['legend'],
                    'description': meas_r['name'] + ' ' + meas_type,
                   }
        samples = []

        # Run through all revisions in our data
        for meta, test_data in data:
            if (not test in test_data['tests'] or
                    not meas in test_data['tests'][test]['measurements']):
                samples.append(measurement_stats(None))
                continue
            test_i = test_data['tests'][test]
            meas_i = test_i['measurements'][meas]
            commit_num = get_data_item(meta, 'layers.meta.commit_count')
            samples.append(measurement_stats(meas_i))
            samples[-1]['commit_num'] = commit_num

        absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean'])
        reldiff = absdiff * 100 / samples[id_comp]['mean']
        new_meas['absdiff'] = absdiff
        new_meas['absdiff_str'] = str(absdiff) if absdiff < 0 else '+' + str(absdiff)
        new_meas['reldiff'] = reldiff
        new_meas['reldiff_str'] = "{:+.1f} %".format(reldiff)
        new_meas['samples'] = samples
        new_meas['value'] = samples[-1]
        new_meas['value_type'] = samples[-1]['val_cls']

        # Compare buildstats
        bs_key = test + '.' + meas
        rev = metadata['commit_num']['value']
        comp_rev = metadata['commit_num']['value_old']
        if (rev in buildstats and bs_key in buildstats[rev] and
                comp_rev in buildstats and bs_key in buildstats[comp_rev]):
            new_meas['buildstats'] = BSSummary(buildstats[comp_rev][bs_key],
                                               buildstats[rev][bs_key])


        new_test['measurements'].append(new_meas)
    tests.append(new_test)

# Chart options
chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'),
                        'max': get_data_item(data[-1][0], 'layers.meta.commit_count')}
             }

print(html.template.render(title="Build Perf Test Report",
                           metadata=metadata, test_data=tests,
                           chart_opts=chart_opts))

def get_buildstats(repo, notes_ref, revs, outdir=None): """Get the buildstats from git notes""" full_ref = 'refs/notes/' + notes_ref if not repo.rev_parse(full_ref): log.error("No buildstats found, please try running " "'git fetch origin %s:%s' to fetch them from the remote", full_ref, full_ref) return

missing = False
buildstats = {}
log.info("Parsing buildstats from 'refs/notes/%s'", notes_ref)
for rev in revs:
    buildstats[rev.commit_number] = {}
    log.debug('Dumping buildstats for %s (%s)', rev.commit_number,
              rev.commit)
    for tag in rev.tags:
        log.debug('    %s', tag)
        try:
            bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref,
                                              'show', tag + '^0']))
        except GitError:
            log.warning("Buildstats not found for %s", tag)
            bs_all = {}
            missing = True

        for measurement, bs in bs_all.items():
            # Write out onto disk
            if outdir:
                tag_base, run_id = tag.rsplit('/', 1)
                tag_base = tag_base.replace('/', '_')
                bs_dir = os.path.join(outdir, measurement, tag_base)
                if not os.path.exists(bs_dir):
                    os.makedirs(bs_dir)
                with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f:
                    json.dump(bs, f, indent=2)

            # Read buildstats into a dict
            _bs = BuildStats.from_json(bs)
            if measurement not in buildstats[rev.commit_number]:
                buildstats[rev.commit_number][measurement] = _bs
            else:
                buildstats[rev.commit_number][measurement].aggregate(_bs)

if missing:
    log.info("Buildstats were missing for some test runs, please "
             "run 'git fetch origin %s:%s' and try again",
             full_ref, full_ref)

return buildstats

def auto_args(repo, args): """Guess arguments, if not defined by the user""" # Get the latest commit in the repo log.debug("Guessing arguments from the latest commit") msg = repo.run_cmd(['log', '-1', '--branches', '--remotes', '--format=%b']) for line in msg.splitlines(): split = line.split(':', 1) if len(split) != 2: continue

    key = split[0]
    val = split[1].strip()
    if key == 'hostname' and not args.hostname:
        log.debug("Using hostname %s", val)
        args.hostname = val
    elif key == 'branch' and not args.branch:
        log.debug("Using branch %s", val)
        args.branch = val

def parse_args(argv): """Parse command line arguments""" description = """ Examine build performance test results from a Git repository""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description=description)

parser.add_argument('--debug', '-d', action='store_true',
                    help="Verbose logging")
parser.add_argument('--repo', '-r', required=True,
                    help="Results repository (local git clone)")
parser.add_argument('--list', '-l', action='count',
                    help="List available test runs")
parser.add_argument('--html', action='store_true',
                    help="Generate report in html format")
group = parser.add_argument_group('Tag and revision')
group.add_argument('--tag-name', '-t',
                   default='{hostname}/{branch}/{machine}/{commit_number}-g{commit}/{tag_number}',
                   help="Tag name (pattern) for finding results")
group.add_argument('--hostname', '-H')
group.add_argument('--branch', '-B', default='master')
group.add_argument('--machine', default='qemux86')
group.add_argument('--history-length', default=25, type=int,
                   help="Number of tested revisions to plot in html report")
group.add_argument('--commit',
                   help="Revision to search for")
group.add_argument('--commit-number',
                   help="Revision number to search for, redundant if "
                        "--commit is specified")
group.add_argument('--commit2',
                   help="Revision to compare with")
group.add_argument('--commit-number2',
                   help="Revision number to compare with, redundant if "
                        "--commit2 is specified")
parser.add_argument('--dump-buildstats', nargs='?', const='.',
                    help="Dump buildstats of the tests")

return parser.parse_args(argv)

def main(argv=None): """Script entry point""" args = parse_args(argv) if args.debug: log.setLevel(logging.DEBUG)

repo = GitRepo(args.repo)

if args.list:
    list_test_revs(repo, args.tag_name, args.list, hostname=args.hostname)
    return 0

# Determine hostname which to use
if not args.hostname:
    auto_args(repo, args)

revs = get_test_revs(repo, args.tag_name, hostname=args.hostname,
                     branch=args.branch, machine=args.machine)
if len(revs) < 2:
    log.error("%d tester revisions found, unable to generate report",
              len(revs))
    return 1

# Pick revisions
if args.commit:
    if args.commit_number:
        log.warning("Ignoring --commit-number as --commit was specified")
    index1 = rev_find(revs, 'commit', args.commit)
elif args.commit_number:
    index1 = rev_find(revs, 'commit_number', args.commit_number)
else:
    index1 = len(revs) - 1

if args.commit2:
    if args.commit_number2:
        log.warning("Ignoring --commit-number2 as --commit2 was specified")
    index2 = rev_find(revs, 'commit', args.commit2)
elif args.commit_number2:
    index2 = rev_find(revs, 'commit_number', args.commit_number2)
else:
    if index1 > 0:
        index2 = index1 - 1
    else:
        log.error("Unable to determine the other commit, use "
                  "--commit2 or --commit-number2 to specify it")
        return 1

index_l = min(index1, index2)
index_r = max(index1, index2)

rev_l = revs[index_l]
rev_r = revs[index_r]
log.debug("Using 'left' revision %s (%s), %s test runs:\n    %s",
          rev_l.commit_number, rev_l.commit, len(rev_l.tags),
          '\n    '.join(rev_l.tags))
log.debug("Using 'right' revision %s (%s), %s test runs:\n    %s",
          rev_r.commit_number, rev_r.commit, len(rev_r.tags),
          '\n    '.join(rev_r.tags))

# Check report format used in the repo (assume all reports in the same fmt)
xml = is_xml_format(repo, revs[index_r].tags[-1])

if args.html:
    index_0 = max(0, min(index_l, index_r - args.history_length))
    rev_range = range(index_0, index_r + 1)
else:
    # We do not need range of commits for text report (no graphs)
    index_0 = index_l
    rev_range = (index_l, index_r)

# Read raw data
log.debug("Reading %d revisions, starting from %s (%s)",
          len(rev_range), revs[index_0].commit_number, revs[index_0].commit)
raw_data = [read_results(repo, revs[i].tags, xml) for i in rev_range]

data = []
for raw_m, raw_d in raw_data:
    data.append(AggregateTestData(aggregate_metadata(raw_m),
                                  aggregate_data(raw_d)))

# Read buildstats only when needed
buildstats = None
if args.dump_buildstats or args.html:
    outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None
    notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch,
                                             args.machine)
    buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir)

# Print report
if not args.html:
    print_diff_report(data[0].metadata, data[0].results,
                      data[1].metadata, data[1].results)
else:
    # Re-map 'left' list index to the data table where index_0 maps to 0
    print_html_report(data, index_l - index_0, buildstats)

return 0

if name == "main": sys.exit(main())