mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
ffae400179
This adds SPDX license headers in place of the wide assortment of things currently in our script headers. We default to GPL-2.0-only except for the oeqa code where it was clearly submitted and marked as MIT on the most part or some scripts which had the "or later" GPL versioning. The patch also drops other obsolete bits of file headers where they were encoountered such as editor modelines, obsolete maintainer information or the phrase "All rights reserved" which is now obsolete and not required in copyright headers (in this case its actually confusing for licensing as all rights were not reserved). More work is needed for OE-Core but this takes care of the bulk of the scripts and meta/lib directories. The top level LICENSE files are tweaked to match the new structure and the SPDX naming. (From OE-Core rev: f8c9c511b5f1b7dbd45b77f345cb6c048ae6763e) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
266 lines
15 KiB
Python
266 lines
15 KiB
Python
#
|
|
# Copyright (C) 2015 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: MIT
|
|
#
|
|
|
|
# Provides functions to help with exporting binaries obtained from built targets
|
|
|
|
import os, re, glob as g, shutil as sh,sys
|
|
from time import sleep
|
|
from .commands import runCmd
|
|
from difflib import SequenceMatcher as SM
|
|
|
|
try:
|
|
import bb
|
|
except ImportError:
|
|
class my_log():
|
|
def __init__(self):
|
|
pass
|
|
def plain(self, msg):
|
|
if msg:
|
|
print(msg)
|
|
def warn(self, msg):
|
|
if msg:
|
|
print("WARNING: " + msg)
|
|
def fatal(self, msg):
|
|
if msg:
|
|
print("FATAL:" + msg)
|
|
sys.exit(1)
|
|
bb = my_log()
|
|
|
|
|
|
def determine_if_poky_env():
|
|
"""
|
|
used to determine if we are inside the poky env or not. Usefull for remote machine where poky is not present
|
|
"""
|
|
check_env = True if ("/scripts" and "/bitbake/bin") in os.getenv("PATH") else False
|
|
return check_env
|
|
|
|
|
|
def get_dest_folder(tune_features, folder_list):
|
|
"""
|
|
Function to determine what rpm deploy dir to choose for a given architecture based on TUNE_FEATURES
|
|
"""
|
|
features_list = tune_features.split(" ")
|
|
features_list.reverse()
|
|
features_list = "_".join(features_list)
|
|
match_rate = 0
|
|
best_match = None
|
|
for folder in folder_list:
|
|
curr_match_rate = SM(None, folder, features_list).ratio()
|
|
if curr_match_rate > match_rate:
|
|
match_rate = curr_match_rate
|
|
best_match = folder
|
|
return best_match
|
|
|
|
|
|
def process_binaries(d, params):
|
|
param_list = params
|
|
export_env = d.getVar("TEST_EXPORT_ONLY")
|
|
|
|
def extract_binary(pth_to_pkg, dest_pth=None):
|
|
cpio_command = runCmd("which cpio")
|
|
rpm2cpio_command = runCmd("ls /usr/bin/rpm2cpio")
|
|
if (cpio_command.status != 0) and (rpm2cpio_command.status != 0):
|
|
bb.fatal("Either \"rpm2cpio\" or \"cpio\" tools are not available on your system."
|
|
"All binaries extraction processes will not be available, crashing all related tests."
|
|
"Please install them according to your OS recommendations") # will exit here
|
|
if dest_pth:
|
|
os.chdir(dest_pth)
|
|
else:
|
|
os.chdir("%s" % os.sep)# this is for native package
|
|
extract_bin_command = runCmd("%s %s | %s -idm" % (rpm2cpio_command.output, pth_to_pkg, cpio_command.output)) # semi-hardcoded because of a bug on poky's rpm2cpio
|
|
return extract_bin_command
|
|
|
|
if determine_if_poky_env(): # machine with poky environment
|
|
exportpath = d.getVar("TEST_EXPORT_DIR") if export_env else d.getVar("DEPLOY_DIR")
|
|
rpm_deploy_dir = d.getVar("DEPLOY_DIR_RPM")
|
|
arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(rpm_deploy_dir))
|
|
arch_rpm_dir = os.path.join(rpm_deploy_dir, arch)
|
|
extracted_bin_dir = os.path.join(exportpath,"binaries", arch, "extracted_binaries")
|
|
packaged_bin_dir = os.path.join(exportpath,"binaries", arch, "packaged_binaries")
|
|
# creating necessary directory structure in case testing is done in poky env.
|
|
if export_env == "0":
|
|
if not os.path.exists(extracted_bin_dir): bb.utils.mkdirhier(extracted_bin_dir)
|
|
if not os.path.exists(packaged_bin_dir): bb.utils.mkdirhier(packaged_bin_dir)
|
|
|
|
if param_list[3] == "native":
|
|
if export_env == "1": #this is a native package and we only need to copy it. no need for extraction
|
|
native_rpm_dir = os.path.join(rpm_deploy_dir, get_dest_folder("{} nativesdk".format(d.getVar("BUILD_SYS")), os.listdir(rpm_deploy_dir)))
|
|
native_rpm_file_list = [item for item in os.listdir(native_rpm_dir) if re.search("nativesdk-" + param_list[0] + "-([0-9]+\.*)", item)]
|
|
if not native_rpm_file_list:
|
|
bb.warn("Couldn't find any version of {} native package. Related tests will most probably fail.".format(param_list[0]))
|
|
return ""
|
|
for item in native_rpm_file_list:# will copy all versions of package. Used version will be selected on remote machine
|
|
bb.plain("Copying native package file: %s" % item)
|
|
sh.copy(os.path.join(rpm_deploy_dir, native_rpm_dir, item), os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries", "native"))
|
|
else: # nothing to do here; running tests under bitbake, so we asume native binaries are in sysroots dir.
|
|
if param_list[1] or param_list[4]:
|
|
bb.warn("Native binary %s %s%s. Running tests under bitbake environment. Version can't be checked except when the test itself does it"
|
|
" and binary can't be removed."%(param_list[0],"has assigned ver. " + param_list[1] if param_list[1] else "",
|
|
", is marked for removal" if param_list[4] else ""))
|
|
else:# the package is target aka DUT intended and it is either required to be delivered in an extracted form or in a packaged version
|
|
target_rpm_file_list = [item for item in os.listdir(arch_rpm_dir) if re.search(param_list[0] + "-([0-9]+\.*)", item)]
|
|
if not target_rpm_file_list:
|
|
bb.warn("Couldn't find any version of target package %s. Please ensure it was built. "
|
|
"Related tests will probably fail." % param_list[0])
|
|
return ""
|
|
if param_list[2] == "rpm": # binary should be deployed as rpm; (other, .deb, .ipk? ; in the near future)
|
|
for item in target_rpm_file_list: # copying all related rpm packages. "Intuition" reasons, someone may need other versions too. Deciding later on version
|
|
bb.plain("Copying target specific packaged file: %s" % item)
|
|
sh.copy(os.path.join(arch_rpm_dir, item), packaged_bin_dir)
|
|
return "copied"
|
|
else: # it is required to extract the binary
|
|
if param_list[1]: # the package is versioned
|
|
for item in target_rpm_file_list:
|
|
if re.match(".*-{}-.*\.rpm".format(param_list[1]), item):
|
|
destination = os.path.join(extracted_bin_dir,param_list[0], param_list[1])
|
|
bb.utils.mkdirhier(destination)
|
|
extract_binary(os.path.join(arch_rpm_dir, item), destination)
|
|
break
|
|
else:
|
|
bb.warn("Couldn't find the desired version %s for target binary %s. Related test cases will probably fail." % (param_list[1], param_list[0]))
|
|
return ""
|
|
return "extracted"
|
|
else: # no version provided, just extract one binary
|
|
destination = os.path.join(extracted_bin_dir,param_list[0],
|
|
re.search(".*-([0-9]+\.[0-9]+)-.*rpm", target_rpm_file_list[0]).group(1))
|
|
bb.utils.mkdirhier(destination)
|
|
extract_binary(os.path.join(arch_rpm_dir, target_rpm_file_list[0]), destination)
|
|
return "extracted"
|
|
else: # remote machine
|
|
binaries_path = os.getenv("bin_dir")# in order to know where the binaries are, bin_dir is set as env. variable
|
|
if param_list[3] == "native": #need to extract the native pkg here
|
|
native_rpm_dir = os.path.join(binaries_path, "native")
|
|
native_rpm_file_list = os.listdir(native_rpm_dir)
|
|
for item in native_rpm_file_list:
|
|
if param_list[1] and re.match("nativesdk-{}-{}-.*\.rpm".format(param_list[0], param_list[1]), item): # native package has version
|
|
extract_binary(os.path.join(native_rpm_dir, item))
|
|
break
|
|
else:# just copy any related native binary
|
|
found_version = re.match("nativesdk-{}-([0-9]+\.[0-9]+)-".format(param_list[0]), item).group(1)
|
|
if found_version:
|
|
extract_binary(os.path.join(native_rpm_dir, item))
|
|
else:
|
|
bb.warn("Couldn't find native package %s%s. Related test cases will be influenced." %
|
|
(param_list[0], " with version " + param_list[1] if param_list[1] else ""))
|
|
return
|
|
|
|
else: # this is for target device
|
|
if param_list[2] == "rpm":
|
|
return "No need to extract, this is an .rpm file"
|
|
arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(binaries_path))
|
|
extracted_bin_path = os.path.join(binaries_path, arch, "extracted_binaries")
|
|
extracted_bin_list = [item for item in os.listdir(extracted_bin_path)]
|
|
packaged_bin_path = os.path.join(binaries_path, arch, "packaged_binaries")
|
|
packaged_bin_file_list = os.listdir(packaged_bin_path)
|
|
# see if the package is already in the extracted ones; maybe it was deployed when exported the env.
|
|
if os.path.exists(os.path.join(extracted_bin_path, param_list[0], param_list[1] if param_list[1] else "")):
|
|
return "binary %s is already extracted" % param_list[0]
|
|
else: # we need to search for it in the packaged binaries directory. It may have been shipped after export
|
|
for item in packaged_bin_file_list:
|
|
if param_list[1]:
|
|
if re.match("%s-%s.*rpm" % (param_list[0], param_list[1]), item): # package with version
|
|
if not os.path.exists(os.path.join(extracted_bin_path, param_list[0],param_list[1])):
|
|
os.makedirs(os.path.join(extracted_bin_path, param_list[0], param_list[1]))
|
|
extract_binary(os.path.join(packaged_bin_path, item), os.path.join(extracted_bin_path, param_list[0],param_list[1]))
|
|
bb.plain("Using {} for {}".format(os.path.join(packaged_bin_path, item), param_list[0]))
|
|
break
|
|
else:
|
|
if re.match("%s-.*rpm" % param_list[0], item):
|
|
found_version = re.match(".*-([0-9]+\.[0-9]+)-", item).group(1)
|
|
if not os.path.exists(os.path.join(extracted_bin_path, param_list[0], found_version)):
|
|
os.makedirs(os.path.join(extracted_bin_path, param_list[0], found_version))
|
|
bb.plain("Used ver. %s for %s" % (found_version, param_list[0]))
|
|
extract_binary(os.path.join(packaged_bin_path, item), os.path.join(extracted_bin_path, param_list[0], found_version))
|
|
break
|
|
else:
|
|
bb.warn("Couldn't find target package %s%s. Please ensure it is available "
|
|
"in either of these directories: extracted_binaries or packaged_binaries. "
|
|
"Related tests will probably fail." % (param_list[0], " with version " + param_list[1] if param_list[1] else ""))
|
|
return
|
|
return "Binary %s extracted successfully." % param_list[0]
|
|
|
|
|
|
def files_to_copy(base_dir):
|
|
"""
|
|
Produces a list of files relative to the base dir path sent as param
|
|
:return: the list of relative path files
|
|
"""
|
|
files_list = []
|
|
dir_list = [base_dir]
|
|
count = 1
|
|
dir_count = 1
|
|
while (dir_count == 1 or dir_count != count):
|
|
count = dir_count
|
|
for dir in dir_list:
|
|
for item in os.listdir(dir):
|
|
if os.path.isdir(os.path.join(dir, item)) and os.path.join(dir, item) not in dir_list:
|
|
dir_list.append(os.path.join(dir, item))
|
|
dir_count = len(dir_list)
|
|
elif os.path.join(dir, item) not in files_list and os.path.isfile(os.path.join(dir, item)):
|
|
files_list.append(os.path.join(dir, item))
|
|
return files_list
|
|
|
|
|
|
def send_bin_to_DUT(d,params):
|
|
from oeqa.oetest import oeRuntimeTest
|
|
param_list = params
|
|
cleanup_list = list()
|
|
bins_dir = os.path.join(d.getVar("TEST_EXPORT_DIR"), "binaries") if determine_if_poky_env() \
|
|
else os.getenv("bin_dir")
|
|
arch = get_dest_folder(d.getVar("TUNE_FEATURES"), os.listdir(bins_dir))
|
|
arch_rpms_dir = os.path.join(bins_dir, arch, "packaged_binaries")
|
|
extracted_bin_dir = os.path.join(bins_dir, arch, "extracted_binaries", param_list[0])
|
|
|
|
def send_extracted_binary():
|
|
bin_local_dir = os.path.join(extracted_bin_dir, param_list[1] if param_list[1] else os.listdir(extracted_bin_dir)[0])
|
|
for item in files_to_copy(bin_local_dir):
|
|
split_path = item.split(bin_local_dir)[1]
|
|
path_on_DUT = split_path if split_path[0] is "/" else "/" + split_path # create the path as on DUT; eg. /usr/bin/bin_file
|
|
(status, output) = oeRuntimeTest.tc.target.copy_to(item, path_on_DUT)
|
|
if status != 0:
|
|
bb.warn("Failed to copy %s binary file %s on the remote target: %s" %
|
|
(param_list[0], "ver. " + param_list[1] if param_list[1] else "", d.getVar("MACHINE")))
|
|
return
|
|
if param_list[4] == "rm":
|
|
cleanup_list.append(path_on_DUT)
|
|
return cleanup_list
|
|
|
|
def send_rpm(remote_path): # if it is not required to have an extracted binary, but to send an .rpm file
|
|
rpm_to_send = ""
|
|
for item in os.listdir(arch_rpms_dir):
|
|
if param_list[1] and re.match("%s-%s-.*rpm"%(param_list[0], param_list[1]), item):
|
|
rpm_to_send = item
|
|
break
|
|
elif re.match("%s-[0-9]+\.[0-9]+-.*rpm" % param_list[0], item):
|
|
rpm_to_send = item
|
|
break
|
|
else:
|
|
bb.warn("No rpm package found for %s %s in .rpm files dir %s. Skipping deployment." %
|
|
(param_list[0], "ver. " + param_list[1] if param_list[1] else "", rpms_file_dir) )
|
|
return
|
|
(status, output) = oeRuntimeTest.tc.target.copy_to(os.path.join(arch_rpms_dir, rpm_to_send), remote_path)
|
|
if status != 0:
|
|
bb.warn("Failed to copy %s on the remote target: %s" %(param_list[0], d.getVar("MACHINE")))
|
|
return
|
|
if param_list[4] == "rm":
|
|
cleanup_list.append(os.path.join(remote_path, rpm_to_send))
|
|
return cleanup_list
|
|
|
|
if param_list[2] == "rpm": # send an .rpm file
|
|
return send_rpm("/home/root") # rpms will be sent on home dir of remote machine
|
|
else:
|
|
return send_extracted_binary()
|
|
|
|
|
|
def rm_bin(removal_list): # need to know both if the binary is sent archived and the path where it is sent if archived
|
|
from oeqa.oetest import oeRuntimeTest
|
|
for item in removal_list:
|
|
(status,output) = oeRuntimeTest.tc.target.run("rm " + item)
|
|
if status != 0:
|
|
bb.warn("Failed to remove: %s. Please ensure connection with the target device is up and running and "
|
|
"you have the needed rights." % item)
|
|
|