mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
oe-build-perf-report/gitarchive: Move common useful functions to library
These functions can be reused by the resulttool code so move to the common function library for this purpose. (From OE-Core rev: c66f848938c04e133259c5b6903dc592866ab385) (From OE-Core rev: 4b1bd35030c5502873106782a35c4f5a9446e20c) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> Signed-off-by: Armin Kuster <akuster808@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
parent
59714c52f0
commit
80adac85bc
|
@ -17,6 +17,8 @@
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
from operator import attrgetter
|
||||||
|
from collections import namedtuple
|
||||||
from oeqa.utils.git import GitRepo, GitError
|
from oeqa.utils.git import GitRepo, GitError
|
||||||
|
|
||||||
class ArchiveError(Exception):
|
class ArchiveError(Exception):
|
||||||
|
@ -171,3 +173,72 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms
|
||||||
log.info("Pushing data to remote")
|
log.info("Pushing data to remote")
|
||||||
data_repo.run_cmd(cmd)
|
data_repo.run_cmd(cmd)
|
||||||
|
|
||||||
|
# Container class for tester revisions
|
||||||
|
TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
|
||||||
|
|
||||||
|
def get_test_runs(log, repo, tag_name, **kwargs):
|
||||||
|
"""Get a sorted list of test runs, matching given pattern"""
|
||||||
|
# First, get field names from the tag name pattern
|
||||||
|
field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
|
||||||
|
undef_fields = [f for f in field_names if f not in kwargs.keys()]
|
||||||
|
|
||||||
|
# Fields for formatting tag name pattern
|
||||||
|
str_fields = dict([(f, '*') for f in field_names])
|
||||||
|
str_fields.update(kwargs)
|
||||||
|
|
||||||
|
# Get a list of all matching tags
|
||||||
|
tag_pattern = tag_name.format(**str_fields)
|
||||||
|
tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
|
||||||
|
log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
|
||||||
|
|
||||||
|
# Parse undefined fields from tag names
|
||||||
|
str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
|
||||||
|
str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
|
||||||
|
str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
|
||||||
|
str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
|
||||||
|
str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
|
||||||
|
# escape parenthesis in fields in order to not messa up the regexp
|
||||||
|
fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
|
||||||
|
str_fields.update(fixed_fields)
|
||||||
|
tag_re = re.compile(tag_name.format(**str_fields))
|
||||||
|
|
||||||
|
# Parse fields from tags
|
||||||
|
revs = []
|
||||||
|
for tag in tags:
|
||||||
|
m = tag_re.match(tag)
|
||||||
|
groups = m.groupdict()
|
||||||
|
revs.append([groups[f] for f in undef_fields] + [tag])
|
||||||
|
|
||||||
|
# Return field names and a sorted list of revs
|
||||||
|
return undef_fields, sorted(revs)
|
||||||
|
|
||||||
|
def get_test_revs(log, repo, tag_name, **kwargs):
|
||||||
|
"""Get list of all tested revisions"""
|
||||||
|
fields, runs = get_test_runs(log, repo, tag_name, **kwargs)
|
||||||
|
|
||||||
|
revs = {}
|
||||||
|
commit_i = fields.index('commit')
|
||||||
|
commit_num_i = fields.index('commit_number')
|
||||||
|
for run in runs:
|
||||||
|
commit = run[commit_i]
|
||||||
|
commit_num = run[commit_num_i]
|
||||||
|
tag = run[-1]
|
||||||
|
if not commit in revs:
|
||||||
|
revs[commit] = TestedRev(commit, commit_num, [tag])
|
||||||
|
else:
|
||||||
|
assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
|
||||||
|
revs[commit].tags.append(tag)
|
||||||
|
|
||||||
|
# Return in sorted table
|
||||||
|
revs = sorted(revs.values(), key=attrgetter('commit_number'))
|
||||||
|
log.debug("Found %d tested revisions:\n %s", len(revs),
|
||||||
|
"\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
|
||||||
|
return revs
|
||||||
|
|
||||||
|
def rev_find(revs, attr, val):
|
||||||
|
"""Search from a list of TestedRev"""
|
||||||
|
for i, rev in enumerate(revs):
|
||||||
|
if getattr(rev, attr) == val:
|
||||||
|
return i
|
||||||
|
raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
|
||||||
|
|
||||||
|
|
|
@ -37,58 +37,18 @@ from buildstats import BuildStats, diff_buildstats, BSVerDiff
|
||||||
scriptpath.add_oe_lib_path()
|
scriptpath.add_oe_lib_path()
|
||||||
|
|
||||||
from oeqa.utils.git import GitRepo, GitError
|
from oeqa.utils.git import GitRepo, GitError
|
||||||
|
import oeqa.utils.gitarchive as gitarchive
|
||||||
|
|
||||||
|
|
||||||
# Setup logging
|
# Setup logging
|
||||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||||
log = logging.getLogger('oe-build-perf-report')
|
log = logging.getLogger('oe-build-perf-report')
|
||||||
|
|
||||||
|
|
||||||
# Container class for tester revisions
|
|
||||||
TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
|
|
||||||
|
|
||||||
|
|
||||||
def get_test_runs(repo, tag_name, **kwargs):
|
|
||||||
"""Get a sorted list of test runs, matching given pattern"""
|
|
||||||
# First, get field names from the tag name pattern
|
|
||||||
field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
|
|
||||||
undef_fields = [f for f in field_names if f not in kwargs.keys()]
|
|
||||||
|
|
||||||
# Fields for formatting tag name pattern
|
|
||||||
str_fields = dict([(f, '*') for f in field_names])
|
|
||||||
str_fields.update(kwargs)
|
|
||||||
|
|
||||||
# Get a list of all matching tags
|
|
||||||
tag_pattern = tag_name.format(**str_fields)
|
|
||||||
tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
|
|
||||||
log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
|
|
||||||
|
|
||||||
# Parse undefined fields from tag names
|
|
||||||
str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
|
|
||||||
str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
|
|
||||||
str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
|
|
||||||
str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
|
|
||||||
str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
|
|
||||||
# escape parenthesis in fields in order to not messa up the regexp
|
|
||||||
fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
|
|
||||||
str_fields.update(fixed_fields)
|
|
||||||
tag_re = re.compile(tag_name.format(**str_fields))
|
|
||||||
|
|
||||||
# Parse fields from tags
|
|
||||||
revs = []
|
|
||||||
for tag in tags:
|
|
||||||
m = tag_re.match(tag)
|
|
||||||
groups = m.groupdict()
|
|
||||||
revs.append([groups[f] for f in undef_fields] + [tag])
|
|
||||||
|
|
||||||
# Return field names and a sorted list of revs
|
|
||||||
return undef_fields, sorted(revs)
|
|
||||||
|
|
||||||
def list_test_revs(repo, tag_name, verbosity, **kwargs):
|
def list_test_revs(repo, tag_name, verbosity, **kwargs):
|
||||||
"""Get list of all tested revisions"""
|
"""Get list of all tested revisions"""
|
||||||
valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
|
valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
|
||||||
|
|
||||||
fields, revs = get_test_runs(repo, tag_name, **valid_kwargs)
|
fields, revs = gitarchive.get_test_runs(log, repo, tag_name, **valid_kwargs)
|
||||||
ignore_fields = ['tag_number']
|
ignore_fields = ['tag_number']
|
||||||
if verbosity < 2:
|
if verbosity < 2:
|
||||||
extra_fields = ['COMMITS', 'TEST RUNS']
|
extra_fields = ['COMMITS', 'TEST RUNS']
|
||||||
|
@ -133,36 +93,6 @@ def list_test_revs(repo, tag_name, verbosity, **kwargs):
|
||||||
|
|
||||||
print_table(rows)
|
print_table(rows)
|
||||||
|
|
||||||
def get_test_revs(repo, tag_name, **kwargs):
|
|
||||||
"""Get list of all tested revisions"""
|
|
||||||
fields, runs = get_test_runs(repo, tag_name, **kwargs)
|
|
||||||
|
|
||||||
revs = {}
|
|
||||||
commit_i = fields.index('commit')
|
|
||||||
commit_num_i = fields.index('commit_number')
|
|
||||||
for run in runs:
|
|
||||||
commit = run[commit_i]
|
|
||||||
commit_num = run[commit_num_i]
|
|
||||||
tag = run[-1]
|
|
||||||
if not commit in revs:
|
|
||||||
revs[commit] = TestedRev(commit, commit_num, [tag])
|
|
||||||
else:
|
|
||||||
assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
|
|
||||||
revs[commit].tags.append(tag)
|
|
||||||
|
|
||||||
# Return in sorted table
|
|
||||||
revs = sorted(revs.values(), key=attrgetter('commit_number'))
|
|
||||||
log.debug("Found %d tested revisions:\n %s", len(revs),
|
|
||||||
"\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
|
|
||||||
return revs
|
|
||||||
|
|
||||||
def rev_find(revs, attr, val):
|
|
||||||
"""Search from a list of TestedRev"""
|
|
||||||
for i, rev in enumerate(revs):
|
|
||||||
if getattr(rev, attr) == val:
|
|
||||||
return i
|
|
||||||
raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
|
|
||||||
|
|
||||||
def is_xml_format(repo, commit):
|
def is_xml_format(repo, commit):
|
||||||
"""Check if the commit contains xml (or json) data"""
|
"""Check if the commit contains xml (or json) data"""
|
||||||
if repo.rev_parse(commit + ':results.xml'):
|
if repo.rev_parse(commit + ':results.xml'):
|
||||||
|
@ -578,11 +508,11 @@ def main(argv=None):
|
||||||
if not args.hostname:
|
if not args.hostname:
|
||||||
auto_args(repo, args)
|
auto_args(repo, args)
|
||||||
|
|
||||||
revs = get_test_revs(repo, args.tag_name, hostname=args.hostname,
|
revs = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
|
||||||
branch=args.branch, machine=args.machine)
|
branch=args.branch, machine=args.machine)
|
||||||
if args.branch2:
|
if args.branch2:
|
||||||
revs2 = get_test_revs(repo, args.tag_name, hostname=args.hostname,
|
revs2 = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
|
||||||
branch=args.branch2, machine=args.machine)
|
branch=args.branch2, machine=args.machine)
|
||||||
if not len(revs2):
|
if not len(revs2):
|
||||||
log.error("No revisions found to compare against")
|
log.error("No revisions found to compare against")
|
||||||
return 1
|
return 1
|
||||||
|
@ -598,9 +528,9 @@ def main(argv=None):
|
||||||
if args.commit:
|
if args.commit:
|
||||||
if args.commit_number:
|
if args.commit_number:
|
||||||
log.warning("Ignoring --commit-number as --commit was specified")
|
log.warning("Ignoring --commit-number as --commit was specified")
|
||||||
index1 = rev_find(revs, 'commit', args.commit)
|
index1 = gitarchive.rev_find(revs, 'commit', args.commit)
|
||||||
elif args.commit_number:
|
elif args.commit_number:
|
||||||
index1 = rev_find(revs, 'commit_number', args.commit_number)
|
index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
|
||||||
else:
|
else:
|
||||||
index1 = len(revs) - 1
|
index1 = len(revs) - 1
|
||||||
|
|
||||||
|
@ -612,9 +542,9 @@ def main(argv=None):
|
||||||
if args.commit2:
|
if args.commit2:
|
||||||
if args.commit_number2:
|
if args.commit_number2:
|
||||||
log.warning("Ignoring --commit-number2 as --commit2 was specified")
|
log.warning("Ignoring --commit-number2 as --commit2 was specified")
|
||||||
index2 = rev_find(revs, 'commit', args.commit2)
|
index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
|
||||||
elif args.commit_number2:
|
elif args.commit_number2:
|
||||||
index2 = rev_find(revs, 'commit_number', args.commit_number2)
|
index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
|
||||||
else:
|
else:
|
||||||
if index1 > 0:
|
if index1 > 0:
|
||||||
index2 = index1 - 1
|
index2 = index1 - 1
|
||||||
|
|
Loading…
Reference in New Issue
Block a user