mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00

We found that manualexecution does not capture test suite values correctly if there are more than one test suite in test cases. After verification has made we found out we should retrieved full test cases value <test_module.test_suite.test_case> from oeqa/manual/ json file rather than split it them into new variables test_suite and test_cases. (From OE-Core rev: db08ec5f48badd5be2fa24cc0d89a4386ab44a59) Signed-off-by: Mazliana <mazliana.mohamad@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
143 lines
6.4 KiB
Python
Executable File
143 lines
6.4 KiB
Python
Executable File
# test case management tool - manual execution from testopia test cases
|
|
#
|
|
# Copyright (c) 2018, Intel Corporation.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms and conditions of the GNU General Public License,
|
|
# version 2, as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import datetime
|
|
import re
|
|
from oeqa.core.runner import OETestResultJSONHelper
|
|
|
|
|
|
def load_json_file(file):
|
|
with open(file, "r") as f:
|
|
return json.load(f)
|
|
|
|
|
|
class ManualTestRunner(object):
|
|
def __init__(self):
|
|
self.jdata = ''
|
|
self.test_module = ''
|
|
self.test_cases_id = ''
|
|
self.configuration = ''
|
|
self.starttime = ''
|
|
self.result_id = ''
|
|
self.write_dir = ''
|
|
|
|
def _get_testcases(self, file):
|
|
self.jdata = load_json_file(file)
|
|
self.test_cases_id = []
|
|
self.test_module = self.jdata[0]['test']['@alias'].split('.', 2)[0]
|
|
for i in self.jdata:
|
|
self.test_cases_id.append(i['test']['@alias'])
|
|
|
|
def _get_input(self, config):
|
|
while True:
|
|
output = input('{} = '.format(config))
|
|
if re.match('^[a-zA-Z0-9_-]+$', output):
|
|
break
|
|
print('Only alphanumeric and underscore/hyphen are allowed. Please try again')
|
|
return output
|
|
|
|
def _create_config(self):
|
|
from oeqa.utils.metadata import get_layers
|
|
from oeqa.utils.commands import get_bb_var
|
|
from resulttool.resultutils import store_map
|
|
|
|
layers = get_layers(get_bb_var('BBLAYERS'))
|
|
self.configuration = {}
|
|
self.configuration['LAYERS'] = layers
|
|
current_datetime = datetime.datetime.now()
|
|
self.starttime = current_datetime.strftime('%Y%m%d%H%M%S')
|
|
self.configuration['STARTTIME'] = self.starttime
|
|
self.configuration['TEST_TYPE'] = 'manual'
|
|
self.configuration['TEST_MODULE'] = self.test_module
|
|
|
|
extra_config = set(store_map['manual']) - set(self.configuration)
|
|
for config in sorted(extra_config):
|
|
print('---------------------------------------------')
|
|
print('This is configuration #%s. Please provide configuration value(use "None" if not applicable).'
|
|
% config)
|
|
print('---------------------------------------------')
|
|
value_conf = self._get_input('Configuration Value')
|
|
print('---------------------------------------------\n')
|
|
self.configuration[config] = value_conf
|
|
|
|
def _create_result_id(self):
|
|
self.result_id = 'manual_' + self.test_module + '_' + self.starttime
|
|
|
|
def _execute_test_steps(self, test_id):
|
|
test_result = {}
|
|
total_steps = len(self.jdata[test_id]['test']['execution'].keys())
|
|
print('------------------------------------------------------------------------')
|
|
print('Executing test case:' + '' '' + self.test_cases_id[test_id])
|
|
print('------------------------------------------------------------------------')
|
|
print('You have total ' + str(total_steps) + ' test steps to be executed.')
|
|
print('------------------------------------------------------------------------\n')
|
|
for step in sorted((self.jdata[test_id]['test']['execution']).keys()):
|
|
print('Step %s: ' % step + self.jdata[test_id]['test']['execution']['%s' % step]['action'])
|
|
print('Expected output: ' + self.jdata[test_id]['test']['execution']['%s' % step]['expected_results'])
|
|
done = input('\nPlease press ENTER when you are done to proceed to next step.\n')
|
|
while True:
|
|
done = input('\nPlease provide test results: (P)assed/(F)ailed/(B)locked/(S)kipped? \n')
|
|
done = done.lower()
|
|
result_types = {'p':'PASSED',
|
|
'f':'FAILED',
|
|
'b':'BLOCKED',
|
|
's':'SKIPPED'}
|
|
if done in result_types:
|
|
for r in result_types:
|
|
if done == r:
|
|
res = result_types[r]
|
|
if res == 'FAILED':
|
|
log_input = input('\nPlease enter the error and the description of the log: (Ex:log:211 Error Bitbake)\n')
|
|
test_result.update({self.test_cases_id[test_id]: {'status': '%s' % res, 'log': '%s' % log_input}})
|
|
else:
|
|
test_result.update({self.test_cases_id[test_id]: {'status': '%s' % res}})
|
|
break
|
|
print('Invalid input!')
|
|
return test_result
|
|
|
|
def _create_write_dir(self):
|
|
basepath = os.environ['BUILDDIR']
|
|
self.write_dir = basepath + '/tmp/log/manual/'
|
|
|
|
def run_test(self, file):
|
|
self._get_testcases(file)
|
|
self._create_config()
|
|
self._create_result_id()
|
|
self._create_write_dir()
|
|
test_results = {}
|
|
print('\nTotal number of test cases in this test suite: ' + '%s\n' % len(self.jdata))
|
|
for i in range(0, len(self.jdata)):
|
|
test_result = self._execute_test_steps(i)
|
|
test_results.update(test_result)
|
|
return self.configuration, self.result_id, self.write_dir, test_results
|
|
|
|
def manualexecution(args, logger):
|
|
testrunner = ManualTestRunner()
|
|
get_configuration, get_result_id, get_write_dir, get_test_results = testrunner.run_test(args.file)
|
|
resultjsonhelper = OETestResultJSONHelper()
|
|
resultjsonhelper.dump_testresult_file(get_write_dir, get_configuration, get_result_id,
|
|
get_test_results)
|
|
return 0
|
|
|
|
def register_commands(subparsers):
|
|
"""Register subcommands from this plugin"""
|
|
parser_build = subparsers.add_parser('manualexecution', help='helper script for results populating during manual test execution.',
|
|
description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/',
|
|
group='manualexecution')
|
|
parser_build.set_defaults(func=manualexecution)
|
|
parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
|