mirror of
git://git.yoctoproject.org/yocto-autobuilder-helper.git
synced 2025-07-19 20:59:02 +02:00
72b22dc785
This ensures that release branch performance build tests work on the correct branch and that comparisions between mut+master and master-next+master start to work correctly. It also ensures reports for the current revision are generated even if later revisions are built. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
214 lines
8.0 KiB
Python
Executable File
214 lines
8.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
#
|
|
# Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.
|
|
#
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import errno
|
|
|
|
import utils
|
|
|
|
parser = utils.ArgParser(description='Runs configurations in json.conf.')
|
|
|
|
parser.add_argument('target',
|
|
help="The 'nightly' target the autobuilder is running")
|
|
parser.add_argument('builddir',
|
|
help="The target build directory to configure")
|
|
parser.add_argument('branchname',
|
|
help="The poky branch name the build is running on")
|
|
parser.add_argument('reponame',
|
|
help="The name of the repository the build is running on")
|
|
parser.add_argument('-s', '--sstateprefix',
|
|
default='',
|
|
help="The directory prefix to publish sstate into")
|
|
parser.add_argument('-b', '--buildappsrcrev',
|
|
default='',
|
|
help="A build appliance SRCREV to use")
|
|
parser.add_argument('-p', '--publish-dir',
|
|
action='store',
|
|
help="Where to publish artefacts to (optional)")
|
|
parser.add_argument('-r', '--results-dir',
|
|
action='store',
|
|
help="Where to publish test results to (optional)")
|
|
parser.add_argument('-u', '--build-url',
|
|
action='store',
|
|
help="URL back to this build (for the error reporting system)")
|
|
parser.add_argument('-t', '--test',
|
|
action='store_true',
|
|
default=False,
|
|
help="Test mode - perform setup and dry-run of commands only")
|
|
parser.add_argument('-q', '--quietlogging',
|
|
action='store_true',
|
|
default=False,
|
|
help="Quiet mode - don't echo bitbake logs to stdout")
|
|
|
|
args = parser.parse_args()
|
|
|
|
scriptsdir = os.path.dirname(os.path.realpath(__file__))
|
|
os.environ["SCRIPTSDIR"] = scriptsdir
|
|
ourconfig = utils.loadconfig()
|
|
ourconfig["HELPERBUILDDIR"] = args.builddir
|
|
ourconfig["HELPERTARGET"] = args.target
|
|
ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "")
|
|
ourconfig["HELPERREPONAME"] = args.reponame
|
|
ourconfig["HELPERBRANCHNAME"] = args.branchname
|
|
|
|
hp = utils.HeaderPrinter()
|
|
|
|
testmode = args.test
|
|
|
|
# Find out the number of steps this target has
|
|
maxsteps = 1
|
|
if args.target in ourconfig['overrides']:
|
|
for v in ourconfig['overrides'][args.target]:
|
|
if v.startswith("step"):
|
|
n = int(v[4:])
|
|
if n <= maxsteps:
|
|
continue
|
|
maxsteps = n
|
|
|
|
hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))
|
|
|
|
finalret = 0
|
|
|
|
def flush():
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
|
|
lognum = 0
|
|
def logname(path, stepnum, logsuffix):
|
|
global lognum
|
|
lognum += 1
|
|
return path + "/command.log.%s%s" % (stepnum, logsuffix)
|
|
|
|
utils.mkdir(args.builddir)
|
|
|
|
revision = "unknown"
|
|
report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision)
|
|
errordir = utils.errorreportdir(args.builddir)
|
|
utils.mkdir(errordir)
|
|
|
|
errorlogs = set()
|
|
|
|
def bitbakecmd(builddir, cmd, report, stepnum, logsuffix, oeenv=True):
|
|
global finalret
|
|
flush()
|
|
log = logname(builddir, stepnum, logsuffix)
|
|
errordir = utils.errorreportdir(builddir)
|
|
try:
|
|
numreports = len(os.listdir(errordir))
|
|
except FileNotFoundError:
|
|
numreports = 0
|
|
|
|
if oeenv:
|
|
cmd = ". ./oe-init-build-env; %s" % cmd
|
|
|
|
if testmode:
|
|
print("Would run '%s'" % cmd)
|
|
return
|
|
|
|
print("Running '%s' with output to %s" % (cmd, log))
|
|
flush()
|
|
|
|
with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p, open(log, 'ab') as f:
|
|
for line in p.stdout:
|
|
if not args.quietlogging:
|
|
sys.stdout.buffer.write(line)
|
|
sys.stdout.flush()
|
|
f.write(line)
|
|
f.flush()
|
|
ret = p.wait()
|
|
if ret:
|
|
hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
|
|
# No error report was written but the command failed so we should write one
|
|
try:
|
|
finalnumreports = len(os.listdir(errordir))
|
|
except FileNotFoundError:
|
|
finalnumreports = 0
|
|
if finalnumreports == numreports:
|
|
report.create(cmd, stepnum, log)
|
|
finalret += 1
|
|
errorlogs.add(log)
|
|
|
|
def runcmd(cmd, *args, **kwargs):
|
|
if testmode:
|
|
print("Running %s" % cmd)
|
|
if "setup-config" not in cmd[0]:
|
|
return
|
|
try:
|
|
subprocess.check_call(cmd, *args, **kwargs)
|
|
except subprocess.CalledProcessError:
|
|
print("ERROR: Command %s failed" % cmd)
|
|
|
|
bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname)
|
|
if bh_path:
|
|
runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])
|
|
|
|
for stepnum in range(1, maxsteps + 1):
|
|
# Add any layers specified
|
|
layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum)
|
|
for layer in layers:
|
|
bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, 'a')
|
|
|
|
flush()
|
|
# Generate the configuration files needed for this step
|
|
if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
|
|
runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])
|
|
|
|
# Execute the targets for this configuration
|
|
targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
|
|
if targets:
|
|
hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
|
|
bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, 'b')
|
|
|
|
# Execute the sanity targets for this configuration
|
|
sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
|
|
if sanitytargets:
|
|
hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
|
|
bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, 'c')
|
|
|
|
# Run any extra commands specified
|
|
cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
|
|
for cmd in cmds:
|
|
hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
|
|
bitbakecmd(args.builddir, cmd, report, stepnum, 'd')
|
|
cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
|
|
for cmd in cmds:
|
|
hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
|
|
bitbakecmd(args.builddir, cmd, report, stepnum, 'd', oeenv=False)
|
|
|
|
# Remove any layers we added in a reverse order
|
|
for layer in reversed(layers):
|
|
bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, 'a')
|
|
|
|
if args.publish_dir:
|
|
hp.printheader("Running publish artefacts")
|
|
runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target])
|
|
|
|
if args.results_dir:
|
|
hp.printheader("Running results collection")
|
|
runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])
|
|
|
|
if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum):
|
|
hp.printheader("Sending any error reports")
|
|
runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url])
|
|
|
|
# Clean up our build directory if things were successful and we're not publishing anything
|
|
# (keep published builds around for longer just in case we need them)
|
|
if not finalret and not args.publish_dir:
|
|
runcmd([scriptsdir + "/../janitor/clobberdir", args.builddir])
|
|
else:
|
|
# Rename any completed build directory so that other builds can't reference paths within it
|
|
runcmd(["mv", args.builddir, args.builddir + "-renamed"])
|
|
|
|
if finalret:
|
|
hp.printheader("There were %s failures" % finalret)
|
|
hp.printheader("Failures in logfiles: %s" % " ".join(errorlogs))
|
|
sys.exit(1)
|
|
|
|
sys.exit(0)
|
|
|