mirror of
git://git.yoctoproject.org/yocto-autobuilder-helper.git
synced 2025-07-19 20:59:02 +02:00
bbe9741d01
summarize_top_output.py is used to summarize the top output that is captured during autobuilder intermittent failures. Use the script to summarize the host top output and publish the summary that is created instead of the raw logfile. Signed-off-by: Sakib Sajal <sakib.sajal@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
353 lines
13 KiB
Python
Executable File
353 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.
|
|
#
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import errno
|
|
|
|
import utils
|
|
|
|
parser = utils.ArgParser(description='Runs configurations in json.conf.')
|
|
|
|
parser.add_argument('target',
|
|
help="The 'nightly' target the autobuilder is running")
|
|
parser.add_argument('builddir',
|
|
help="The target build directory to configure")
|
|
parser.add_argument('branchname',
|
|
help="The poky branch name the build is running on")
|
|
parser.add_argument('reponame',
|
|
help="The name of the repository the build is running on")
|
|
parser.add_argument('-s', '--sstateprefix',
|
|
default='',
|
|
help="The directory prefix to publish sstate into")
|
|
parser.add_argument('-b', '--buildappsrcrev',
|
|
default='',
|
|
help="A build appliance SRCREV to use")
|
|
parser.add_argument('-p', '--publish-dir',
|
|
action='store',
|
|
help="Where to publish artefacts to (optional)")
|
|
parser.add_argument('-r', '--results-dir',
|
|
action='store',
|
|
help="Where to publish test results to (optional)")
|
|
parser.add_argument('-u', '--build-url',
|
|
action='store',
|
|
help="URL back to this build (for the error reporting system)")
|
|
parser.add_argument('--build-type',
|
|
action='store',
|
|
default="quick",
|
|
help="the type of build being triggered (full or quick)")
|
|
parser.add_argument('-t', '--test',
|
|
action='store_true',
|
|
default=False,
|
|
help="Test mode - perform setup and dry-run of commands only")
|
|
parser.add_argument('-q', '--quietlogging',
|
|
action='store_true',
|
|
default=False,
|
|
help="Quiet mode - don't echo bitbake logs to stdout")
|
|
parser.add_argument('--workername',
|
|
action='store',
|
|
default=None,
|
|
help="the name of the worker the build is running on")
|
|
parser.add_argument('-j', '--json-outputfile',
|
|
action='store',
|
|
default="",
|
|
help="the file to store json information about the build in")
|
|
parser.add_argument('--stepname',
|
|
action='store',
|
|
default=None,
|
|
help="the name of the step to run")
|
|
parser.add_argument('--phase',
|
|
action='store',
|
|
default=None,
|
|
help="the phase of the step to run")
|
|
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
scriptsdir = os.path.dirname(os.path.realpath(__file__))
|
|
os.environ["SCRIPTSDIR"] = scriptsdir
|
|
ourconfig = utils.loadconfig()
|
|
ourconfig["HELPERBUILDDIR"] = args.builddir
|
|
ourconfig["HELPERTARGET"] = args.target
|
|
ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "")
|
|
ourconfig["HELPERREPONAME"] = args.reponame
|
|
ourconfig["HELPERBRANCHNAME"] = args.branchname
|
|
|
|
hp = utils.HeaderPrinter()
|
|
|
|
testmode = args.test
|
|
|
|
# toolchain tests are run in system mode for x86, user mode for the other
|
|
# arches due to speed
|
|
# toolchain tests only run on full builds
|
|
if args.build_type == "quick":
|
|
ourconfig["HELPERSTMACHTARGS"] = "-a -t machine"
|
|
elif args.build_type == "full":
|
|
if args.target == "qemux86" or args.target == "qemux86-64":
|
|
ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-system"
|
|
else:
|
|
ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-user"
|
|
|
|
# Find out the number of steps this target has
|
|
maxsteps = 0
|
|
stepnum = 0
|
|
if args.target in ourconfig['overrides']:
|
|
maxsteps = 1
|
|
for v in ourconfig['overrides'][args.target]:
|
|
if v.startswith("step"):
|
|
n = int(v[4:])
|
|
if n <= maxsteps:
|
|
continue
|
|
maxsteps = n
|
|
|
|
hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))
|
|
|
|
jcfg = False
|
|
if args.json_outputfile:
|
|
jsonconfig = []
|
|
jcfg = True
|
|
|
|
# There is a 50 char limit on "bbname" but buildbot may append "_1", "_2" if multiple steps
|
|
# with the same name exist in a build
|
|
def addentry(name, description, phase):
|
|
jsonconfig.append({"name" : name, "bbname" : description[:46], "phase" : phase, "description" : description})
|
|
|
|
def addstepentry(name, taskdesc, shortname, description, detail, phase):
|
|
bbname = taskdesc
|
|
if shortname:
|
|
bbname = shortname + ": " + taskdesc
|
|
bbdesc = taskdesc
|
|
if description:
|
|
bbdesc = description
|
|
if detail:
|
|
bbdesc = bbdesc + ": " + detail
|
|
jsonconfig.append({"name" : name, "bbname" : bbname[:46], "phase" : phase, "description" : bbdesc})
|
|
|
|
if jcfg:
|
|
buildtools = utils.setup_buildtools_tarball(ourconfig, args.workername, None, checkonly=True)
|
|
if buildtools:
|
|
addentry("buildtools", "Setup buildtools tarball", "init")
|
|
else:
|
|
utils.setup_buildtools_tarball(ourconfig, args.workername, args.builddir + "/../buildtools")
|
|
if args.phase == "init" and args.stepname == "buildtools":
|
|
sys.exit(0)
|
|
|
|
logconfig = args.builddir + "/../bitbake/contrib/autobuilderlog.json"
|
|
print("Using BB_LOGCONFIG=%s" % logconfig)
|
|
os.environ["BB_LOGCONFIG"] = logconfig
|
|
|
|
finalret = 0
|
|
|
|
def flush():
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
|
|
def logname(path, stepnum, stepname):
|
|
return path + "/command-%s-%s.log" % (stepnum, stepname)
|
|
|
|
utils.mkdir(args.builddir)
|
|
|
|
revision = "unknown"
|
|
report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision)
|
|
errordir = utils.errorreportdir(args.builddir)
|
|
utils.mkdir(errordir)
|
|
|
|
errorlogs = set()
|
|
|
|
def bitbakecmd(builddir, cmd, report, stepnum, stepname, oeenv=True):
|
|
global finalret
|
|
flush()
|
|
log = logname(builddir, stepnum, stepname)
|
|
errordir = utils.errorreportdir(builddir)
|
|
try:
|
|
numreports = len(os.listdir(errordir))
|
|
except FileNotFoundError:
|
|
numreports = 0
|
|
|
|
def writelog(msg, a, b):
|
|
a.write(msg)
|
|
b.write(msg)
|
|
|
|
if oeenv:
|
|
cmd = ". ./oe-init-build-env; %s" % cmd
|
|
|
|
if testmode:
|
|
print("Would run '%s'" % cmd)
|
|
return
|
|
|
|
with open(log, "a") as outf:
|
|
writelog("Running '%s' with output to %s\n" % (cmd, log), outf, sys.stdout)
|
|
|
|
autoconf = builddir + "/conf/auto.conf"
|
|
if os.path.exists(autoconf):
|
|
with open(autoconf, "r") as inf, open(log, "a") as outf:
|
|
writelog("auto.conf settings:\n", outf, sys.stdout)
|
|
for line in inf.readlines():
|
|
writelog(line, outf, sys.stdout)
|
|
writelog("\n", outf, sys.stdout)
|
|
|
|
flush()
|
|
|
|
with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0) as p, open(log, 'ab') as f:
|
|
for line in p.stdout:
|
|
writelog(line, f, sys.stdout.buffer)
|
|
sys.stdout.flush()
|
|
f.flush()
|
|
ret = p.wait()
|
|
if ret:
|
|
hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
|
|
# No error report was written but the command failed so we should write one
|
|
try:
|
|
finalnumreports = len(os.listdir(errordir))
|
|
except FileNotFoundError:
|
|
finalnumreports = 0
|
|
if finalnumreports == numreports:
|
|
report.create(cmd, stepnum, log)
|
|
finalret += 1
|
|
errorlogs.add(log)
|
|
|
|
def runcmd(cmd, *args, **kwargs):
|
|
if testmode:
|
|
print("Running %s" % cmd)
|
|
if "setup-config" not in cmd[0]:
|
|
return
|
|
try:
|
|
subprocess.check_call(cmd, *args, **kwargs)
|
|
except subprocess.CalledProcessError:
|
|
print("ERROR: Command %s failed" % cmd)
|
|
|
|
bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname, 1)
|
|
if bh_path:
|
|
if jcfg:
|
|
addentry("buildhistory-init", "Initialize buildhistory", "init")
|
|
if args.phase == "init" and args.stepname == "buildhistory-init":
|
|
if bh_path:
|
|
runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])
|
|
sys.exit(0)
|
|
|
|
def handle_stepnum(stepnum):
|
|
shortdesc = utils.getconfigvar("shortname", ourconfig, args.target, stepnum) or ""
|
|
desc = utils.getconfigvar("description", ourconfig, args.target, stepnum) or ""
|
|
|
|
# Add any layers specified
|
|
layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum)
|
|
if jcfg:
|
|
if layers:
|
|
addstepentry("add-layers", "Add layers", shortdesc, desc, str(layers), str(stepnum))
|
|
elif args.stepname == "add-layers":
|
|
for layer in layers:
|
|
bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, args.stepname)
|
|
|
|
flush()
|
|
|
|
# Generate the configuration files needed for this step
|
|
if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
|
|
if jcfg:
|
|
addstepentry("write-config", "Write config", shortdesc, desc, None, str(stepnum))
|
|
elif args.stepname == "write-config":
|
|
runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])
|
|
|
|
# Execute the targets for this configuration
|
|
targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
|
|
if targets:
|
|
if jcfg:
|
|
addstepentry("build-targets", "Build targets", shortdesc, desc, str(targets), str(stepnum))
|
|
elif args.stepname == "build-targets":
|
|
hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
|
|
bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, args.stepname)
|
|
|
|
# Execute the sanity targets for this configuration
|
|
sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
|
|
if sanitytargets:
|
|
if jcfg:
|
|
addstepentry("test-targets", "QA targets", shortdesc, desc, str(sanitytargets), str(stepnum))
|
|
elif args.stepname == "test-targets":
|
|
hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
|
|
bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, args.stepname)
|
|
|
|
# Run any extra commands specified
|
|
cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
|
|
if jcfg:
|
|
if cmds:
|
|
addstepentry("cmds", "Run cmds", shortdesc, desc, str(cmds), str(stepnum))
|
|
elif args.stepname == "cmds":
|
|
for cmd in cmds:
|
|
hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
|
|
bitbakecmd(args.builddir, cmd, report, stepnum, args.stepname)
|
|
|
|
cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
|
|
if jcfg:
|
|
if cmds:
|
|
addstepentry("plain-cmds", "Run cmds", shortdesc, desc, str(cmds), str(stepnum))
|
|
elif args.stepname == "plain-cmds":
|
|
for cmd in cmds:
|
|
hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
|
|
bitbakecmd(args.builddir, cmd, report, stepnum, args.stepname, oeenv=False)
|
|
|
|
if jcfg:
|
|
if layers:
|
|
addstepentry("remove-layers", "Remove layers", shortdesc, desc, str(layers), str(stepnum))
|
|
elif args.stepname == "remove-layers":
|
|
# Remove any layers we added in a reverse order
|
|
for layer in reversed(layers):
|
|
bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, args.stepname)
|
|
|
|
if not jcfg:
|
|
sys.exit(finalret)
|
|
|
|
if jcfg:
|
|
for stepnum in range(1, maxsteps + 1):
|
|
handle_stepnum(stepnum)
|
|
else:
|
|
try:
|
|
stepnum = int(args.phase)
|
|
except ValueError:
|
|
stepnum = None
|
|
|
|
if stepnum is not None:
|
|
handle_stepnum(stepnum)
|
|
|
|
|
|
if jcfg:
|
|
addentry("publish", "Publishing artefacts", "finish")
|
|
elif args.phase == "finish" and args.stepname == "publish":
|
|
if args.publish_dir:
|
|
hp.printheader("Running publish artefacts")
|
|
runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target])
|
|
sys.exit(0)
|
|
|
|
if jcfg:
|
|
addentry("collect-results", "Collecting result files", "finish")
|
|
elif args.phase == "finish" and args.stepname == "collect-results":
|
|
if args.results_dir:
|
|
hp.printheader("Running results collection")
|
|
runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])
|
|
runcmd([scriptsdir + "/summarize_top_output.py", args.results_dir, args.target])
|
|
sys.exit(0)
|
|
|
|
if jcfg:
|
|
addentry("send-errors", "Sending error reports", "finish")
|
|
elif args.phase == "finish" and args.stepname == "send-errors":
|
|
if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum):
|
|
hp.printheader("Sending any error reports")
|
|
runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url])
|
|
sys.exit(0)
|
|
|
|
if jcfg:
|
|
addentry("builddir-cleanup", "Cleaning up build directory", "finish")
|
|
elif args.phase == "finish" and args.stepname == "builddir-cleanup":
|
|
if args.builddir and os.path.exists(args.builddir):
|
|
runcmd(["mv", args.builddir, args.builddir + "-renamed"])
|
|
|
|
if args.json_outputfile:
|
|
with open(args.json_outputfile, "w") as f:
|
|
json.dump(jsonconfig, f, indent=4, sort_keys=True)
|
|
|
|
sys.exit(0)
|
|
|