yocto-autobuilder-helper/scripts/run-config

#!/usr/bin/env python3

Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.

import json import os import sys import subprocess import errno import fnmatch import glob

import utils

parser = utils.ArgParser(description='Runs configurations in json.conf.')

parser.add_argument('target', help="The 'nightly' target the autobuilder is running") parser.add_argument('builddir', help="The target build directory to configure") parser.add_argument('branchname', help="The poky branch name the build is running on") parser.add_argument('reponame', help="The name of the repository the build is running on") parser.add_argument('-s', '--sstateprefix', default='', help="The directory prefix to publish sstate into") parser.add_argument('-b', '--buildappsrcrev', default='', help="A build appliance SRCREV to use") parser.add_argument('-p', '--publish-dir', action='store', help="Where to publish artefacts to (optional)") parser.add_argument('-r', '--results-dir', action='store', help="Where to publish test results to (optional)") parser.add_argument('-u', '--build-url', action='store', help="URL back to this build (for the error reporting system)") parser.add_argument('--build-type', action='store', default="quick", help="the type of build being triggered (full or quick)") parser.add_argument('-t', '--test', action='store_true', default=False, help="Test mode - perform setup and dry-run of commands only") parser.add_argument('-q', '--quietlogging', action='store_true', default=False, help="Quiet mode - don't echo bitbake logs to stdout") parser.add_argument('--workername', action='store', default=None, help="the name of the worker the build is running on")

args = parser.parse_args()

scriptsdir = os.path.dirname(os.path.realpath(file)) os.environ["SCRIPTSDIR"] = scriptsdir ourconfig = utils.loadconfig() ourconfig["HELPERBUILDDIR"] = args.builddir ourconfig["HELPERTARGET"] = args.target ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "") ourconfig["HELPERREPONAME"] = args.reponame ourconfig["HELPERBRANCHNAME"] = args.branchname

hp = utils.HeaderPrinter()

testmode = args.test

toolchain tests are run in system mode for x86, user mode for the other

arches due to speed

toolchain tests only run on full builds

if args.build_type == "quick": ourconfig["HELPERSTMACHTARGS"] = "-a -t machine" elif args.build_type == "full": if args.target == "qemux86" or args.target == "qemux86-64": ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-system" else: ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-user"

Find out the number of steps this target has

maxsteps = 1 if args.target in ourconfig['overrides']: for v in ourconfig['overrides'][args.target]: if v.startswith("step"): n = int(v[4:]) if n <= maxsteps: continue maxsteps = n

hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))

bttarball = None if "buildtools" in ourconfig and args.workername: btcfg = utils.getconfig("buildtools", ourconfig) for entry in btcfg: if fnmatch.fnmatch(args.workername, entry): bttarball = btcfg[entry] break

btenv = None if bttarball: btdir = args.builddir + "../buildtools" if not os.path.exists(btdir): btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball) if not os.path.exists(btdlpath): os.makedirs(os.path.dirname(btdlpath), exist_ok=True) subprocess.check_call(["wget", "-O", btdlpath, bttarball]) os.chmod(btdlpath, 0o775) subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"]) btenv = glob.glob(btdir + "/environment-setup*") # We either parse or wrap all our execution calls, rock and a hard place :( with open(btenv[0], "r") as f: for line in f.readlines(): if line.startswith("export "): line = line.split(" ", 1)[1].split("=", 1) if "$PATH" in line[1]: line[1] = line[1].replace("$PATH", os.environ["PATH"]) os.environ[line[0]] = line[1]

finalret = 0

def flush(): sys.stdout.flush() sys.stderr.flush()

lognum = 0 def logname(path, stepnum, logsuffix): global lognum lognum += 1 return path + "/command.log.%s%s" % (stepnum, logsuffix)

utils.mkdir(args.builddir)

revision = "unknown" report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision) errordir = utils.errorreportdir(args.builddir) utils.mkdir(errordir)

errorlogs = set()

def bitbakecmd(builddir, cmd, report, stepnum, logsuffix, oeenv=True): global finalret flush() log = logname(builddir, stepnum, logsuffix) errordir = utils.errorreportdir(builddir) try: numreports = len(os.listdir(errordir)) except FileNotFoundError: numreports = 0

if oeenv:
    cmd = ". ./oe-init-build-env; %s" % cmd

if testmode:
    print("Would run '%s'" % cmd)
    return

print("Running '%s' with output to %s" % (cmd, log))
flush()

autoconf = builddir + "/conf/auto.conf"
if os.path.exists(autoconf):
    with open(autoconf, "r") as inf, open(log, "a") as outf:
        outf.write("auto.conf settings:\n")
        for line in inf.readlines():
            outf.write(line)
        outf.write("\n")

with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p, open(log, 'ab') as f:
    for line in p.stdout:
        if not args.quietlogging:
            sys.stdout.buffer.write(line)
            sys.stdout.flush()
        f.write(line)
        f.flush()
    ret = p.wait()
if ret:
    hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
    # No error report was written but the command failed so we should write one
    try:
        finalnumreports = len(os.listdir(errordir))
    except FileNotFoundError:
        finalnumreports = 0
    if finalnumreports == numreports:
        report.create(cmd, stepnum, log)
    finalret += 1
    errorlogs.add(log)

def runcmd(cmd, *args, **kwargs): if testmode: print("Running %s" % cmd) if "setup-config" not in cmd[0]: return try: subprocess.check_call(cmd, *args, **kwargs) except subprocess.CalledProcessError: print("ERROR: Command %s failed" % cmd)

bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname, 1) if bh_path: runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])

for stepnum in range(1, maxsteps + 1): # Add any layers specified layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum) for layer in layers: bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, 'a')

flush()
# Generate the configuration files needed for this step
if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
    runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])

# Execute the targets for this configuration
targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
if targets:
    hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
    bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, 'b')

# Execute the sanity targets for this configuration
sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
if sanitytargets:
    hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
    bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, 'c')

# Run any extra commands specified
cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
for cmd in cmds:
    hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
    bitbakecmd(args.builddir, cmd, report, stepnum, 'd')
cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
for cmd in cmds:
    hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
    bitbakecmd(args.builddir, cmd, report, stepnum, 'd', oeenv=False)

# Remove any layers we added in a reverse order
for layer in reversed(layers):
    bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, 'a')

if args.publish_dir: hp.printheader("Running publish artefacts") runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target])

if args.results_dir: hp.printheader("Running results collection") runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])

if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum): hp.printheader("Sending any error reports") runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url])

if args.builddir and os.path.exists(args.builddir): # Clean up our build directory if things were successful and we're not publishing anything # (keep published builds around for longer just in case we need them) if not finalret and not args.publish_dir: runcmd([scriptsdir + "/../janitor/clobberdir", args.builddir]) else: # Rename any completed build directory so that other builds can't reference paths within it runcmd(["mv", args.builddir, args.builddir + "-renamed"])

if finalret: hp.printheader("There were %s failures" % finalret) hp.printheader("Failures in logfiles: %s" % " ".join(errorlogs)) sys.exit(1)

sys.exit(0)