mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
10b6fbfb8a
This is a combined patch of the various tweaks and improvements I made to resulttool: * Avoid subprocess.run() as its a python 3.6 feature and we have autobuilder workers with 3.5. * Avoid python keywords as variable names * Simplify dict accesses using .get() * Rename resultsutils -> resultutils to match the resultstool -> resulttool rename * Formalised the handling of "file_name" to "TESTSERIES" which the code will now add into the json configuration data if its not present, based on the directory name. * When we don't have failed test cases, print something saying so instead of an empty table * Tweak the table headers in the report to be more readable (reference "Test Series" instead if file_id and ID instead of results_id) * Improve/simplify the max string length handling * Merge the counts and percentage data into one table in the report since printing two reports of the same data confuses the user * Removed the confusing header in the regression report * Show matches, then regressions, then unmatched runs in the regression report, also remove chatting unneeded output * Try harder to "pair" up matching configurations to reduce noise in the regressions report * Abstracted the "mapping" table concept used to pairing in the regression code to general code in resultutils * Created multiple mappings for results analysis, results storage and 'flattening' results data in a merge * Simplify the merge command to take a source and a destination, letting the destination be a directory or a file, removing the need for an output directory parameter * Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression mappings * Have the store command place the testresults files in a layout from the mapping, making commits into the git repo for results storage more useful for simple comparison purposes * Set the oe-git-archive tag format appropriately for oeqa results storage (and simplify the commit messages closer to their defaults) * Fix oe-git-archive to use the commit/branch data from the results file * Cleaned up the command option help to match other changes * Follow the model of git branch/tag processing used by oe-build-perf-report and use that to read the data using git show to avoid branch change * Add ptest summary to the report command * Update the tests to match the above changes (From OE-Core rev: b4513e75f746a0989b09ee53cb85e489d41e5783) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
193 lines
8.4 KiB
Python
193 lines
8.4 KiB
Python
# resulttool - regression analysis
|
|
#
|
|
# Copyright (c) 2019, Intel Corporation.
|
|
# Copyright (c) 2019, Linux Foundation
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms and conditions of the GNU General Public License,
|
|
# version 2, as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
import resulttool.resultutils as resultutils
|
|
import json
|
|
|
|
from oeqa.utils.git import GitRepo
|
|
import oeqa.utils.gitarchive as gitarchive
|
|
|
|
def compare_result(logger, base_name, target_name, base_result, target_result):
|
|
base_result = base_result.get('result')
|
|
target_result = target_result.get('result')
|
|
result = {}
|
|
if base_result and target_result:
|
|
for k in base_result:
|
|
base_testcase = base_result[k]
|
|
base_status = base_testcase.get('status')
|
|
if base_status:
|
|
target_testcase = target_result.get(k, {})
|
|
target_status = target_testcase.get('status')
|
|
if base_status != target_status:
|
|
result[k] = {'base': base_status, 'target': target_status}
|
|
else:
|
|
logger.error('Failed to retrieved base test case status: %s' % k)
|
|
if result:
|
|
resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
|
|
for k in result:
|
|
resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
|
|
else:
|
|
resultstring = "Match: %s\n %s" % (base_name, target_name)
|
|
return result, resultstring
|
|
|
|
def get_results(logger, source):
|
|
return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
|
|
|
|
def regression(args, logger):
|
|
base_results = get_results(logger, args.base_result)
|
|
target_results = get_results(logger, args.target_result)
|
|
|
|
regression_common(args, logger, base_results, target_results)
|
|
|
|
def regression_common(args, logger, base_results, target_results):
|
|
if args.base_result_id:
|
|
base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
|
|
if args.target_result_id:
|
|
target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
|
|
|
|
matches = []
|
|
regressions = []
|
|
notfound = []
|
|
|
|
for a in base_results:
|
|
if a in target_results:
|
|
base = list(base_results[a].keys())
|
|
target = list(target_results[a].keys())
|
|
# We may have multiple base/targets which are for different configurations. Start by
|
|
# removing any pairs which match
|
|
for c in base.copy():
|
|
for b in target.copy():
|
|
res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
|
|
if not res:
|
|
matches.append(resstr)
|
|
base.remove(c)
|
|
target.remove(b)
|
|
break
|
|
# Should only now see regressions, we may not be able to match multiple pairs directly
|
|
for c in base:
|
|
for b in target:
|
|
res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
|
|
if res:
|
|
regressions.append(resstr)
|
|
else:
|
|
notfound.append("%s not found in target" % a)
|
|
print("\n".join(matches))
|
|
print("\n".join(regressions))
|
|
print("\n".join(notfound))
|
|
|
|
return 0
|
|
|
|
def regression_git(args, logger):
|
|
base_results = {}
|
|
target_results = {}
|
|
|
|
tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
|
|
repo = GitRepo(args.repo)
|
|
|
|
revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)
|
|
|
|
if args.branch2:
|
|
revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
|
|
if not len(revs2):
|
|
logger.error("No revisions found to compare against")
|
|
return 1
|
|
if not len(revs):
|
|
logger.error("No revision to report on found")
|
|
return 1
|
|
else:
|
|
if len(revs) < 2:
|
|
logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
|
|
return 1
|
|
|
|
# Pick revisions
|
|
if args.commit:
|
|
if args.commit_number:
|
|
logger.warning("Ignoring --commit-number as --commit was specified")
|
|
index1 = gitarchive.rev_find(revs, 'commit', args.commit)
|
|
elif args.commit_number:
|
|
index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
|
|
else:
|
|
index1 = len(revs) - 1
|
|
|
|
if args.branch2:
|
|
revs2.append(revs[index1])
|
|
index1 = len(revs2) - 1
|
|
revs = revs2
|
|
|
|
if args.commit2:
|
|
if args.commit_number2:
|
|
logger.warning("Ignoring --commit-number2 as --commit2 was specified")
|
|
index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
|
|
elif args.commit_number2:
|
|
index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
|
|
else:
|
|
if index1 > 0:
|
|
index2 = index1 - 1
|
|
# Find the closest matching commit number for comparision
|
|
# In future we could check the commit is a common ancestor and
|
|
# continue back if not but this good enough for now
|
|
while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
|
|
index2 = index2 - 1
|
|
else:
|
|
logger.error("Unable to determine the other commit, use "
|
|
"--commit2 or --commit-number2 to specify it")
|
|
return 1
|
|
|
|
logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))
|
|
|
|
base_results = resultutils.git_get_result(repo, revs[index1][2])
|
|
target_results = resultutils.git_get_result(repo, revs[index2][2])
|
|
|
|
regression_common(args, logger, base_results, target_results)
|
|
|
|
return 0
|
|
|
|
def register_commands(subparsers):
|
|
"""Register subcommands from this plugin"""
|
|
|
|
parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
|
|
description='regression analysis comparing the base set of results to the target results',
|
|
group='analysis')
|
|
parser_build.set_defaults(func=regression)
|
|
parser_build.add_argument('base_result',
|
|
help='base result file/directory for the comparison')
|
|
parser_build.add_argument('target_result',
|
|
help='target result file/directory to compare with')
|
|
parser_build.add_argument('-b', '--base-result-id', default='',
|
|
help='(optional) filter the base results to this result ID')
|
|
parser_build.add_argument('-t', '--target-result-id', default='',
|
|
help='(optional) filter the target results to this result ID')
|
|
|
|
parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
|
|
description='regression analysis comparing base result set to target '
|
|
'result set',
|
|
group='analysis')
|
|
parser_build.set_defaults(func=regression_git)
|
|
parser_build.add_argument('repo',
|
|
help='the git repository containing the data')
|
|
parser_build.add_argument('-b', '--base-result-id', default='',
|
|
help='(optional) default select regression based on configurations unless base result '
|
|
'id was provided')
|
|
parser_build.add_argument('-t', '--target-result-id', default='',
|
|
help='(optional) default select regression based on configurations unless target result '
|
|
'id was provided')
|
|
|
|
parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
|
|
parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
|
|
parser_build.add_argument('--commit', help="Revision to search for")
|
|
parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
|
|
parser_build.add_argument('--commit2', help="Revision to compare with")
|
|
parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
|
|
|