mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 12:59:02 +02:00
resulttool: modify to be multi-machine
Currently, the code will sum all of the different machine results into a single report of the tests results. This can lead to confusion as to which machine may be experiencing issues. Modify the code to store the results in a per machine basis and report them accordingly. (From OE-Core rev: 16d4031ea5df8a4ddfdb937d35464c09e1abd10e) Signed-off-by: Jon Mason <jdmason@kudzu.us> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Jon Mason
Richard Purdie
parent
c113a83dd5
commit
cb153e8da8
|
|
@ -56,7 +56,7 @@ class ResultToolTests(OESelftestTestCase):
|
|||
'test4': {'status': 'ERROR'},
|
||||
'test5': {'status': 'SKIPPED'}}}
|
||||
report = ResultsTextReport()
|
||||
result_report = report.get_aggregated_test_result(None, result_data)
|
||||
result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine')
|
||||
self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed'])
|
||||
self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed'])
|
||||
self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped'])
|
||||
|
|
|
|||
|
|
@ -24,16 +24,19 @@ class ResultsTextReport(object):
|
|||
'skipped': ['SKIPPED', 'skipped']}
|
||||
|
||||
|
||||
def handle_ptest_result(self, k, status, result):
|
||||
def handle_ptest_result(self, k, status, result, machine):
|
||||
if machine not in self.ptests:
|
||||
self.ptests[machine] = {}
|
||||
|
||||
if k == 'ptestresult.sections':
|
||||
# Ensure tests without any test results still show up on the report
|
||||
for suite in result['ptestresult.sections']:
|
||||
if suite not in self.ptests:
|
||||
self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if suite not in self.ptests[machine]:
|
||||
self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if 'duration' in result['ptestresult.sections'][suite]:
|
||||
self.ptests[suite]['duration'] = result['ptestresult.sections'][suite]['duration']
|
||||
self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration']
|
||||
if 'timeout' in result['ptestresult.sections'][suite]:
|
||||
self.ptests[suite]['duration'] += " T"
|
||||
self.ptests[machine][suite]['duration'] += " T"
|
||||
return
|
||||
try:
|
||||
_, suite, test = k.split(".", 2)
|
||||
|
|
@ -47,22 +50,25 @@ class ResultsTextReport(object):
|
|||
suite = suite + "." + suite1
|
||||
except ValueError:
|
||||
pass
|
||||
if suite not in self.ptests:
|
||||
self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if suite not in self.ptests[machine]:
|
||||
self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
for tk in self.result_types:
|
||||
if status in self.result_types[tk]:
|
||||
self.ptests[suite][tk] += 1
|
||||
self.ptests[machine][suite][tk] += 1
|
||||
|
||||
def handle_ltptest_result(self, k, status, result, machine):
|
||||
if machine not in self.ltptests:
|
||||
self.ltptests[machine] = {}
|
||||
|
||||
def handle_ltptest_result(self, k, status, result):
|
||||
if k == 'ltpresult.sections':
|
||||
# Ensure tests without any test results still show up on the report
|
||||
for suite in result['ltpresult.sections']:
|
||||
if suite not in self.ltptests:
|
||||
self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if suite not in self.ltptests[machine]:
|
||||
self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if 'duration' in result['ltpresult.sections'][suite]:
|
||||
self.ltptests[suite]['duration'] = result['ltpresult.sections'][suite]['duration']
|
||||
self.ltptests[machine][suite]['duration'] = result['ltpresult.sections'][suite]['duration']
|
||||
if 'timeout' in result['ltpresult.sections'][suite]:
|
||||
self.ltptests[suite]['duration'] += " T"
|
||||
self.ltptests[machine][suite]['duration'] += " T"
|
||||
return
|
||||
try:
|
||||
_, suite, test = k.split(".", 2)
|
||||
|
|
@ -77,20 +83,23 @@ class ResultsTextReport(object):
|
|||
suite = suite + "." + suite1
|
||||
except ValueError:
|
||||
pass
|
||||
if suite not in self.ltptests:
|
||||
self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if suite not in self.ltptests[machine]:
|
||||
self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
for tk in self.result_types:
|
||||
if status in self.result_types[tk]:
|
||||
self.ltptests[suite][tk] += 1
|
||||
self.ltptests[machine][suite][tk] += 1
|
||||
|
||||
def handle_ltpposixtest_result(self, k, status, result, machine):
|
||||
if machine not in self.ltpposixtests:
|
||||
self.ltpposixtests[machine] = {}
|
||||
|
||||
def handle_ltpposixtest_result(self, k, status, result):
|
||||
if k == 'ltpposixresult.sections':
|
||||
# Ensure tests without any test results still show up on the report
|
||||
for suite in result['ltpposixresult.sections']:
|
||||
if suite not in self.ltpposixtests:
|
||||
self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if suite not in self.ltpposixtests[machine]:
|
||||
self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if 'duration' in result['ltpposixresult.sections'][suite]:
|
||||
self.ltpposixtests[suite]['duration'] = result['ltpposixresult.sections'][suite]['duration']
|
||||
self.ltpposixtests[machine][suite]['duration'] = result['ltpposixresult.sections'][suite]['duration']
|
||||
return
|
||||
try:
|
||||
_, suite, test = k.split(".", 2)
|
||||
|
|
@ -104,19 +113,13 @@ class ResultsTextReport(object):
|
|||
suite = suite + "." + suite1
|
||||
except ValueError:
|
||||
pass
|
||||
if suite not in self.ltpposixtests:
|
||||
self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
if suite not in self.ltpposixtests[machine]:
|
||||
self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
|
||||
for tk in self.result_types:
|
||||
if status in self.result_types[tk]:
|
||||
self.ltpposixtests[suite][tk] += 1
|
||||
self.ltpposixtests[machine][suite][tk] += 1
|
||||
|
||||
def get_aggregated_test_result(self, logger, testresult):
|
||||
test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
|
||||
def get_aggregated_test_result(self, logger, testresult):
|
||||
test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
|
||||
def get_aggregated_test_result(self, logger, testresult):
|
||||
test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
|
||||
def get_aggregated_test_result(self, logger, testresult):
|
||||
def get_aggregated_test_result(self, logger, testresult, machine):
|
||||
test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
|
||||
result = testresult.get('result', [])
|
||||
for k in result:
|
||||
|
|
@ -127,11 +130,11 @@ class ResultsTextReport(object):
|
|||
if test_status in self.result_types['failed']:
|
||||
test_count_report['failed_testcases'].append(k)
|
||||
if k.startswith("ptestresult."):
|
||||
self.handle_ptest_result(k, test_status, result)
|
||||
self.handle_ptest_result(k, test_status, result, machine)
|
||||
if k.startswith("ltpresult."):
|
||||
self.handle_ltptest_result(k, test_status, result)
|
||||
self.handle_ltptest_result(k, test_status, result, machine)
|
||||
if k.startswith("ltpposixresult."):
|
||||
self.handle_ltpposixtest_result(k, test_status, result)
|
||||
self.handle_ltpposixtest_result(k, test_status, result, machine)
|
||||
return test_count_report
|
||||
|
||||
def print_test_report(self, template_file_name, test_count_reports):
|
||||
|
|
@ -141,10 +144,8 @@ class ResultsTextReport(object):
|
|||
env = Environment(loader=file_loader, trim_blocks=True)
|
||||
template = env.get_template(template_file_name)
|
||||
havefailed = False
|
||||
haveptest = bool(self.ptests)
|
||||
haveltp = bool(self.ltptests)
|
||||
haveltpposix = bool(self.ltpposixtests)
|
||||
reportvalues = []
|
||||
machines = []
|
||||
cols = ['passed', 'failed', 'skipped']
|
||||
maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0}
|
||||
for line in test_count_reports:
|
||||
|
|
@ -162,21 +163,24 @@ class ResultsTextReport(object):
|
|||
reportvalues.append(vals)
|
||||
if line['failed_testcases']:
|
||||
havefailed = True
|
||||
for ptest in self.ptests:
|
||||
if len(ptest) > maxlen['ptest']:
|
||||
maxlen['ptest'] = len(ptest)
|
||||
for ltptest in self.ltptests:
|
||||
if len(ltptest) > maxlen['ltptest']:
|
||||
maxlen['ltptest'] = len(ltptest)
|
||||
for ltpposixtest in self.ltpposixtests:
|
||||
if len(ltpposixtest) > maxlen['ltpposixtest']:
|
||||
maxlen['ltpposixtest'] = len(ltpposixtest)
|
||||
if line['machine'] not in machines:
|
||||
machines.append(line['machine'])
|
||||
for (machine, report) in self.ptests.items():
|
||||
for ptest in self.ptests[machine]:
|
||||
if len(ptest) > maxlen['ptest']:
|
||||
maxlen['ptest'] = len(ptest)
|
||||
for (machine, report) in self.ltptests.items():
|
||||
for ltptest in self.ltptests[machine]:
|
||||
if len(ltptest) > maxlen['ltptest']:
|
||||
maxlen['ltptest'] = len(ltptest)
|
||||
for (machine, report) in self.ltpposixtests.items():
|
||||
for ltpposixtest in self.ltpposixtests[machine]:
|
||||
if len(ltpposixtest) > maxlen['ltpposixtest']:
|
||||
maxlen['ltpposixtest'] = len(ltpposixtest)
|
||||
output = template.render(reportvalues=reportvalues,
|
||||
havefailed=havefailed,
|
||||
haveptest=haveptest,
|
||||
machines=machines,
|
||||
ptests=self.ptests,
|
||||
haveltp=haveltp,
|
||||
haveltpposix=haveltpposix,
|
||||
ltptests=self.ltptests,
|
||||
ltpposixtests=self.ltpposixtests,
|
||||
maxlen=maxlen)
|
||||
|
|
@ -200,7 +204,9 @@ class ResultsTextReport(object):
|
|||
for testsuite in testresults:
|
||||
for resultid in testresults[testsuite]:
|
||||
result = testresults[testsuite][resultid]
|
||||
test_count_report = self.get_aggregated_test_result(logger, result)
|
||||
machine = result['configuration']['MACHINE']
|
||||
test_count_report = self.get_aggregated_test_result(logger, result, machine)
|
||||
test_count_report['machine'] = machine
|
||||
test_count_report['testseries'] = result['configuration']['TESTSERIES']
|
||||
test_count_report['result_id'] = resultid
|
||||
test_count_reports.append(test_count_report)
|
||||
|
|
|
|||
|
|
@ -9,54 +9,62 @@ Test Result Status Summary (Counts/Percentages sorted by testseries, ID)
|
|||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% if haveptest %}
|
||||
|
||||
{% for machine in machines %}
|
||||
{% if ptests[machine] %}
|
||||
==============================================================================================================
|
||||
PTest Result Summary
|
||||
{{ machine }} PTest Result Summary
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for ptest in ptests |sort %}
|
||||
{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[ptest]['duration']|string) }}
|
||||
{% for ptest in ptests[machine] |sort %}
|
||||
{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% else %}
|
||||
There was no ptest data
|
||||
There was no ptest data for {{ machine }}
|
||||
{% endif %}
|
||||
|
||||
{% if haveltp %}
|
||||
{% endfor %}
|
||||
|
||||
|
||||
{% for machine in machines %}
|
||||
{% if ltptests[machine] %}
|
||||
==============================================================================================================
|
||||
Ltp Test Result Summary
|
||||
{{ machine }} Ltp Test Result Summary
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for ltptest in ltptests |sort %}
|
||||
{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[ltptest]['duration']|string) }}
|
||||
{% for ltptest in ltptests[machine] |sort %}
|
||||
{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[machine][ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[machine][ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[machine][ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[machine][ltptest]['duration']|string) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% else %}
|
||||
There was no LTP Test data
|
||||
There was no LTP Test data for {{ machine }}
|
||||
{% endif %}
|
||||
|
||||
{% if haveltpposix %}
|
||||
{% endfor %}
|
||||
|
||||
|
||||
{% for machine in machines %}
|
||||
{% if ltpposixtests[machine] %}
|
||||
==============================================================================================================
|
||||
Ltp Posix Result Summary
|
||||
{{ machine }} Ltp Posix Result Summary
|
||||
==============================================================================================================
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
{% for ltpposixtest in ltpposixtests |sort %}
|
||||
{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[ltpposixtest]['duration']|string) }}
|
||||
{% for ltpposixtest in ltpposixtests[machine] |sort %}
|
||||
{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[machine][ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[machine][ltpposixtest]['duration']|string) }}
|
||||
{% endfor %}
|
||||
--------------------------------------------------------------------------------------------------------------
|
||||
|
||||
{% else %}
|
||||
There was no LTP Posix Test data
|
||||
There was no LTP Posix Test data for {{ machine }}
|
||||
{% endif %}
|
||||
|
||||
{% endfor %}
|
||||
|
||||
|
||||
==============================================================================================================
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user