oe-build-perf-report/gitarchive: Move common useful functions to library

These functions can be reused by the resulttool code so move to the common
function library for this purpose.

(From OE-Core rev: c66f848938c04e133259c5b6903dc592866ab385)

(From OE-Core rev: 4b1bd35030c5502873106782a35c4f5a9446e20c)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2019-02-20 17:07:56 +00:00
parent 59714c52f0
commit 80adac85bc
2 changed files with 81 additions and 80 deletions

View File

@ -17,6 +17,8 @@
import os import os
import re import re
import sys import sys
from operator import attrgetter
from collections import namedtuple
from oeqa.utils.git import GitRepo, GitError from oeqa.utils.git import GitRepo, GitError
class ArchiveError(Exception): class ArchiveError(Exception):
@ -171,3 +173,72 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms
log.info("Pushing data to remote") log.info("Pushing data to remote")
data_repo.run_cmd(cmd) data_repo.run_cmd(cmd)
# Container class for tester revisions
TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
def get_test_runs(log, repo, tag_name, **kwargs):
"""Get a sorted list of test runs, matching given pattern"""
# First, get field names from the tag name pattern
field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
undef_fields = [f for f in field_names if f not in kwargs.keys()]
# Fields for formatting tag name pattern
str_fields = dict([(f, '*') for f in field_names])
str_fields.update(kwargs)
# Get a list of all matching tags
tag_pattern = tag_name.format(**str_fields)
tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
# Parse undefined fields from tag names
str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
# escape parenthesis in fields in order to not messa up the regexp
fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
str_fields.update(fixed_fields)
tag_re = re.compile(tag_name.format(**str_fields))
# Parse fields from tags
revs = []
for tag in tags:
m = tag_re.match(tag)
groups = m.groupdict()
revs.append([groups[f] for f in undef_fields] + [tag])
# Return field names and a sorted list of revs
return undef_fields, sorted(revs)
def get_test_revs(log, repo, tag_name, **kwargs):
"""Get list of all tested revisions"""
fields, runs = get_test_runs(log, repo, tag_name, **kwargs)
revs = {}
commit_i = fields.index('commit')
commit_num_i = fields.index('commit_number')
for run in runs:
commit = run[commit_i]
commit_num = run[commit_num_i]
tag = run[-1]
if not commit in revs:
revs[commit] = TestedRev(commit, commit_num, [tag])
else:
assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
revs[commit].tags.append(tag)
# Return in sorted table
revs = sorted(revs.values(), key=attrgetter('commit_number'))
log.debug("Found %d tested revisions:\n %s", len(revs),
"\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
return revs
def rev_find(revs, attr, val):
"""Search from a list of TestedRev"""
for i, rev in enumerate(revs):
if getattr(rev, attr) == val:
return i
raise ValueError("Unable to find '{}' value '{}'".format(attr, val))

View File

@ -37,58 +37,18 @@ from buildstats import BuildStats, diff_buildstats, BSVerDiff
scriptpath.add_oe_lib_path() scriptpath.add_oe_lib_path()
from oeqa.utils.git import GitRepo, GitError from oeqa.utils.git import GitRepo, GitError
import oeqa.utils.gitarchive as gitarchive
# Setup logging # Setup logging
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
log = logging.getLogger('oe-build-perf-report') log = logging.getLogger('oe-build-perf-report')
# Container class for tester revisions
TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
def get_test_runs(repo, tag_name, **kwargs):
"""Get a sorted list of test runs, matching given pattern"""
# First, get field names from the tag name pattern
field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
undef_fields = [f for f in field_names if f not in kwargs.keys()]
# Fields for formatting tag name pattern
str_fields = dict([(f, '*') for f in field_names])
str_fields.update(kwargs)
# Get a list of all matching tags
tag_pattern = tag_name.format(**str_fields)
tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
# Parse undefined fields from tag names
str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
# escape parenthesis in fields in order to not messa up the regexp
fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
str_fields.update(fixed_fields)
tag_re = re.compile(tag_name.format(**str_fields))
# Parse fields from tags
revs = []
for tag in tags:
m = tag_re.match(tag)
groups = m.groupdict()
revs.append([groups[f] for f in undef_fields] + [tag])
# Return field names and a sorted list of revs
return undef_fields, sorted(revs)
def list_test_revs(repo, tag_name, verbosity, **kwargs): def list_test_revs(repo, tag_name, verbosity, **kwargs):
"""Get list of all tested revisions""" """Get list of all tested revisions"""
valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None]) valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
fields, revs = get_test_runs(repo, tag_name, **valid_kwargs) fields, revs = gitarchive.get_test_runs(log, repo, tag_name, **valid_kwargs)
ignore_fields = ['tag_number'] ignore_fields = ['tag_number']
if verbosity < 2: if verbosity < 2:
extra_fields = ['COMMITS', 'TEST RUNS'] extra_fields = ['COMMITS', 'TEST RUNS']
@ -133,36 +93,6 @@ def list_test_revs(repo, tag_name, verbosity, **kwargs):
print_table(rows) print_table(rows)
def get_test_revs(repo, tag_name, **kwargs):
"""Get list of all tested revisions"""
fields, runs = get_test_runs(repo, tag_name, **kwargs)
revs = {}
commit_i = fields.index('commit')
commit_num_i = fields.index('commit_number')
for run in runs:
commit = run[commit_i]
commit_num = run[commit_num_i]
tag = run[-1]
if not commit in revs:
revs[commit] = TestedRev(commit, commit_num, [tag])
else:
assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
revs[commit].tags.append(tag)
# Return in sorted table
revs = sorted(revs.values(), key=attrgetter('commit_number'))
log.debug("Found %d tested revisions:\n %s", len(revs),
"\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
return revs
def rev_find(revs, attr, val):
"""Search from a list of TestedRev"""
for i, rev in enumerate(revs):
if getattr(rev, attr) == val:
return i
raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
def is_xml_format(repo, commit): def is_xml_format(repo, commit):
"""Check if the commit contains xml (or json) data""" """Check if the commit contains xml (or json) data"""
if repo.rev_parse(commit + ':results.xml'): if repo.rev_parse(commit + ':results.xml'):
@ -578,11 +508,11 @@ def main(argv=None):
if not args.hostname: if not args.hostname:
auto_args(repo, args) auto_args(repo, args)
revs = get_test_revs(repo, args.tag_name, hostname=args.hostname, revs = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
branch=args.branch, machine=args.machine) branch=args.branch, machine=args.machine)
if args.branch2: if args.branch2:
revs2 = get_test_revs(repo, args.tag_name, hostname=args.hostname, revs2 = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
branch=args.branch2, machine=args.machine) branch=args.branch2, machine=args.machine)
if not len(revs2): if not len(revs2):
log.error("No revisions found to compare against") log.error("No revisions found to compare against")
return 1 return 1
@ -598,9 +528,9 @@ def main(argv=None):
if args.commit: if args.commit:
if args.commit_number: if args.commit_number:
log.warning("Ignoring --commit-number as --commit was specified") log.warning("Ignoring --commit-number as --commit was specified")
index1 = rev_find(revs, 'commit', args.commit) index1 = gitarchive.rev_find(revs, 'commit', args.commit)
elif args.commit_number: elif args.commit_number:
index1 = rev_find(revs, 'commit_number', args.commit_number) index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
else: else:
index1 = len(revs) - 1 index1 = len(revs) - 1
@ -612,9 +542,9 @@ def main(argv=None):
if args.commit2: if args.commit2:
if args.commit_number2: if args.commit_number2:
log.warning("Ignoring --commit-number2 as --commit2 was specified") log.warning("Ignoring --commit-number2 as --commit2 was specified")
index2 = rev_find(revs, 'commit', args.commit2) index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
elif args.commit_number2: elif args.commit_number2:
index2 = rev_find(revs, 'commit_number', args.commit_number2) index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
else: else:
if index1 > 0: if index1 > 0:
index2 = index1 - 1 index2 = index1 - 1