yocto-autobuilder-helper/scripts/run-config

#!/usr/bin/env python3

Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.

import json import os import sys import subprocess import errno

import utils

parser = utils.ArgParser(description='Runs configurations in json.conf.')

parser.add_argument('target', help="The 'nightly' target the autobuilder is running") parser.add_argument('builddir', help="The target build directory to configure") parser.add_argument('branchname', help="The poky branch name the build is running on") parser.add_argument('reponame', help="The name of the repository the build is running on") parser.add_argument('-s', '--sstateprefix', default='', help="The directory prefix to publish sstate into") parser.add_argument('-b', '--buildappsrcrev', default='', help="A build appliance SRCREV to use") parser.add_argument('-p', '--publish-dir', action='store', help="Where to publish artefacts to (optional)") parser.add_argument('-r', '--results-dir', action='store', help="Where to publish test results to (optional)") parser.add_argument('-u', '--build-url', action='store', help="URL back to this build (for the error reporting system)") parser.add_argument('-t', '--test', action='store_true', default=False, help="Test mode - perform setup and dry-run of commands only") parser.add_argument('-q', '--quietlogging', action='store_true', default=False, help="Quiet mode - don't echo bitbake logs to stdout")

args = parser.parse_args()

scriptsdir = os.path.dirname(os.path.realpath(file)) os.environ["SCRIPTSDIR"] = scriptsdir ourconfig = utils.loadconfig() ourconfig["HELPERBUILDDIR"] = args.builddir ourconfig["HELPERTARGET"] = args.target ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "") ourconfig["HELPERREPONAME"] = args.reponame ourconfig["HELPERBRANCHNAME"] = args.branchname

hp = utils.HeaderPrinter()

testmode = args.test

Find out the number of steps this target has

maxsteps = 1 if args.target in ourconfig['overrides']: for v in ourconfig['overrides'][args.target]: if v.startswith("step"): n = int(v[4:]) if n <= maxsteps: continue maxsteps = n

hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))

finalret = 0

def flush(): sys.stdout.flush() sys.stderr.flush()

lognum = 0 def logname(path, stepnum, logsuffix): global lognum lognum += 1 return path + "/command.log.%s%s" % (stepnum, logsuffix)

utils.mkdir(args.builddir)

revision = "unknown" report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision) errordir = utils.errorreportdir(args.builddir) utils.mkdir(errordir)

errorlogs = set()

def bitbakecmd(builddir, cmd, report, stepnum, logsuffix, oeenv=True): global finalret flush() log = logname(builddir, stepnum, logsuffix) errordir = utils.errorreportdir(builddir) try: numreports = len(os.listdir(errordir)) except FileNotFoundError: numreports = 0

if oeenv:
    cmd = ". ./oe-init-build-env; %s" % cmd

if testmode:
    print("Would run '%s'" % cmd)
    return

print("Running '%s' with output to %s" % (cmd, log))
flush()

with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p, open(log, 'ab') as f:
    for line in p.stdout:
        if not args.quietlogging:
            sys.stdout.buffer.write(line)
            sys.stdout.flush()
        f.write(line)
        f.flush()
    ret = p.wait()
if ret:
    hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
    # No error report was written but the command failed so we should write one
    try:
        finalnumreports = len(os.listdir(errordir))
    except FileNotFoundError:
        finalnumreports = 0
    if finalnumreports == numreports:
        report.create(cmd, stepnum, log)
    finalret += 1
    errorlogs.add(log)

def runcmd(cmd, *args, **kwargs): if testmode: print("Running %s" % cmd) if "setup-config" not in cmd[0]: return try: subprocess.check_call(cmd, *args, **kwargs) except subprocess.CalledProcessError: print("ERROR: Command %s failed" % cmd)

bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname, stepnum) if bh_path: runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])

for stepnum in range(1, maxsteps + 1): # Add any layers specified layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum) for layer in layers: bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, 'a')

flush()
# Generate the configuration files needed for this step
if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
    runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])

# Execute the targets for this configuration
targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
if targets:
    hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
    bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, 'b')

# Execute the sanity targets for this configuration
sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
if sanitytargets:
    hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
    bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, 'c')

# Run any extra commands specified
cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
for cmd in cmds:
    hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
    bitbakecmd(args.builddir, cmd, report, stepnum, 'd')
cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
for cmd in cmds:
    hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
    bitbakecmd(args.builddir, cmd, report, stepnum, 'd', oeenv=False)

# Remove any layers we added in a reverse order
for layer in reversed(layers):
    bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, 'a')

if args.publish_dir: hp.printheader("Running publish artefacts") runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target])

if args.results_dir: hp.printheader("Running results collection") runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])

if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum): hp.printheader("Sending any error reports") runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url])

Clean up our build directory if things were successful and we're not publishing anything

(keep published builds around for longer just in case we need them)

if not finalret and not args.publish_dir: runcmd([scriptsdir + "/../janitor/clobberdir", args.builddir]) else: # Rename any completed build directory so that other builds can't reference paths within it runcmd(["mv", args.builddir, args.builddir + "-renamed"])

if finalret: hp.printheader("There were %s failures" % finalret) hp.printheader("Failures in logfiles: %s" % " ".join(errorlogs)) sys.exit(1)

sys.exit(0)