mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
1fd5ebdb06
OEQA outputs test results into json files and these files were
archived by Autobuilder during QA releases. Example: each oe-selftest
run by Autobuilder for different host distro generate a
testresults.json file.
These scripts were developed as a test result tools to manage
these testresults.json file.
Using the "store" operation, user can store multiple testresults.json
files as well as the pre-configured directories used to hold those files.
Using the "merge" operation, user can merge multiple testresults.json
files to a target file.
Using the "report" operation, user can view the test result summary
for all available testresults.json files inside a ordinary directory
or a git repository.
Using the "regression-file" operation, user can perform regression
analysis on testresults.json files specified. Using the "regression-dir"
and "regression-git" operations, user can perform regression analysis
on directory and git accordingly.
These resulttool operations expect the testresults.json file to use
the json format below.
{
"<testresult_1>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
...
"<testresult_n>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
}
To use these scripts, first source oe environment, then run the
entry point script to look for help.
$ resulttool
To store test result from oeqa automated tests, execute the below
$ resulttool store <source_dir> <git_branch>
To merge multiple testresults.json files, execute the below
$ resulttool merge <base_result_file> <target_result_file>
To report test report, execute the below
$ resulttool report <source_dir>
To perform regression file analysis, execute the below
$ resulttool regression-file <base_result_file> <target_result_file>
To perform regression dir analysis, execute the below
$ resulttool regression-dir <base_result_dir> <target_result_dir>
To perform regression git analysis, execute the below
$ resulttool regression-git <source_dir> <base_branch> <target_branch>
[YOCTO# 13012]
[YOCTO# 12654]
(From OE-Core rev: 78a322d7be402a5b9b5abf26ad35670a8535408a)
Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
67 lines
2.2 KiB
Python
67 lines
2.2 KiB
Python
# test result tool - utilities
|
|
#
|
|
# Copyright (c) 2019, Intel Corporation.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms and conditions of the GNU General Public License,
|
|
# version 2, as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
import os
|
|
import json
|
|
import scriptpath
|
|
scriptpath.add_oe_lib_path()
|
|
from oeqa.utils.git import GitRepo, GitError
|
|
|
|
def load_json_file(file):
|
|
with open(file, "r") as f:
|
|
return json.load(f)
|
|
|
|
def dump_json_data(write_dir, file_name, json_data):
|
|
file_content = json.dumps(json_data, sort_keys=True, indent=4)
|
|
file_path = os.path.join(write_dir, file_name)
|
|
with open(file_path, 'w') as the_file:
|
|
the_file.write(file_content)
|
|
|
|
def get_dict_value(logger, dict, key):
|
|
try:
|
|
return dict[key]
|
|
except KeyError:
|
|
if logger:
|
|
logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key))
|
|
return None
|
|
except TypeError:
|
|
if logger:
|
|
logger.debug('Faced TypeError exception: dict=%s: key=%s' % (dict, key))
|
|
return None
|
|
|
|
def pop_dict_element(logger, dict, key):
|
|
try:
|
|
dict.pop(key)
|
|
except KeyError:
|
|
if logger:
|
|
logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key))
|
|
except AttributeError:
|
|
if logger:
|
|
logger.debug('Faced AttributeError exception: dict=%s: key=%s' % (dict, key))
|
|
|
|
def checkout_git_dir(git_dir, git_branch):
|
|
try:
|
|
repo = GitRepo(git_dir, is_topdir=True)
|
|
repo.run_cmd('checkout %s' % git_branch)
|
|
return True
|
|
except GitError:
|
|
return False
|
|
|
|
def get_directory_files(source_dir, excludes, file):
|
|
files_in_dir = []
|
|
for root, dirs, files in os.walk(source_dir, topdown=True):
|
|
[dirs.remove(d) for d in list(dirs) if d in excludes]
|
|
for name in files:
|
|
if name == file:
|
|
files_in_dir.append(os.path.join(root, name))
|
|
return files_in_dir |