yocto-autobuilder-helper/scripts/run-config

#!/usr/bin/env python3

Copyright Linux Foundation, Richard Purdie

SPDX-License-Identifier: GPL-2.0-only

Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.

import json import os import sys import subprocess import errno

import utils

parser = utils.ArgParser(description='Runs configurations in json.conf.')

parser.add_argument('target', help="The 'nightly' target the autobuilder is running") parser.add_argument('builddir', help="The target build directory to configure") parser.add_argument('branchname', help="The poky branch name the build is running on") parser.add_argument('reponame', help="The name of the repository the build is running on") parser.add_argument('-s', '--sstateprefix', default='', help="The directory prefix to publish sstate into") parser.add_argument('-b', '--buildappsrcrev', default='', help="A build appliance SRCREV to use") parser.add_argument('-p', '--publish-dir', action='store', help="Where to publish artefacts to (optional)") parser.add_argument('-r', '--results-dir', action='store', help="Where to publish test results to (optional)") parser.add_argument('-u', '--build-url', action='store', help="URL back to this build (for the error reporting system)") parser.add_argument('--build-type', action='store', default="quick", help="the type of build being triggered (full or quick)") parser.add_argument('-t', '--test', action='store_true', default=False, help="Test mode - perform setup and dry-run of commands only") parser.add_argument('-q', '--quietlogging', action='store_true', default=False, help="Quiet mode - don't echo bitbake logs to stdout") parser.add_argument('--workername', action='store', default=None, help="the name of the worker the build is running on") parser.add_argument('-j', '--json-outputfile', action='store', default="", help="the file to store json information about the build in") parser.add_argument('--stepname', action='store', default=None, help="the name of the step to run") parser.add_argument('--phase', action='store', default=None, help="the phase of the step to run")

args = parser.parse_args()

scriptsdir = os.path.dirname(os.path.realpath(file)) os.environ["SCRIPTSDIR"] = scriptsdir ourconfig = utils.loadconfig() ourconfig["HELPERBUILDDIR"] = args.builddir ourconfig["HELPERTARGET"] = args.target ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "") ourconfig["HELPERREPONAME"] = args.reponame ourconfig["HELPERBRANCHNAME"] = args.branchname

hp = utils.HeaderPrinter()

testmode = args.test

toolchain tests are run in system mode for x86, user mode for the other

arches due to speed

arch = args.target.replace("-tc", "") if arch == "qemux86" or arch == "qemux86-64": ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-system" else: ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-user"

Find out the number of steps this target has

maxsteps = 0 stepnum = 0 if args.target in ourconfig['overrides']: maxsteps = 1 for v in ourconfig['overrides'][args.target]: if v.startswith("step"): n = int(v[4:]) if n <= maxsteps: continue maxsteps = n

hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))

jcfg = False if args.json_outputfile: jsonconfig = [] jcfg = True

There is a 50 char limit on "bbname" but buildbot may append "_1", "_2" if multiple steps

with the same name exist in a build

def addentry(name, description, phase): jsonconfig.append({"name" : name, "bbname" : description[:46], "phase" : phase, "description" : description})

def addstepentry(name, taskdesc, shortname, description, detail, phase, usepty=False): bbname = taskdesc if shortname: bbname = shortname + ": " + taskdesc bbdesc = taskdesc if description: bbdesc = description if detail: bbdesc = bbdesc + ": " + detail jsonconfig.append({"name" : name, "bbname" : bbname[:46], "phase" : phase, "description" : bbdesc, "usepty" : usepty})

if jcfg: buildtools = utils.setup_buildtools_tarball(ourconfig, args.workername, None, checkonly=True) if buildtools: addentry("buildtools", "Setup buildtools tarball", "init") else: # If we're executing a specific step, check whether we buildtools is disabled for it buildtools = True if args.stepname in ("build-targets", "cmds", "test-targets", "plain-cmds"): try: buildtools = not utils.getconfigvar("NOBUILDTOOLS", ourconfig, args.target, int(args.phase)) except ValueError: # Not an integer step phase pass

if buildtools:
    utils.setup_buildtools_tarball(ourconfig, args.workername, args.builddir + "/../buildtools")
if args.phase == "init" and args.stepname == "buildtools":
    sys.exit(0)

extratools = utils.getconfigvar("extratools", ourconfig, args.target) if jcfg: if extratools: addentry("extratools", "Setup extratools tarball", "init") elif extratools: utils.setup_tools_tarball(ourconfig, args.builddir + "/../extratools", extratools, "extratools") if args.phase == "init" and args.stepname == "extratools": sys.exit(0)

logconfig = args.builddir + "/../bitbake/contrib/autobuilderlog.json" print("Using BB_LOGCONFIG=%s" % logconfig) os.environ["BB_LOGCONFIG"] = logconfig

finalret = 0

def flush(): sys.stdout.flush() sys.stderr.flush()

def logname(path, stepnum, stepname): return path + "/command-%s-%s.log" % (stepnum, stepname)

utils.mkdir(args.builddir)

revision = "unknown" report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision) errordir = utils.errorreportdir(args.builddir) utils.mkdir(errordir)

errorlogs = set()

def log_file_contents(filename, builddir, stepnum, stepname): logfile = logname(builddir, stepnum, stepname) with open(logfile, "a") as outf, open(filename, "r") as f: def log(s): outf.write(s) sys.stdout.write(s)

    log("Contents of %s:\n" % filename)
    for line in f:
        log(line)
    log("\n")

def bitbakecmd(builddir, cmd, report, stepnum, stepname, oeenv=True): global finalret flush() log = logname(builddir, stepnum, stepname) errordir = utils.errorreportdir(builddir) try: numreports = len(os.listdir(errordir)) except FileNotFoundError: numreports = 0

def writelog(msg, a, b):
    a.write(msg)
    b.write(msg)

if oeenv:
    cmd = ". ./oe-init-build-env; %s" % cmd

if testmode:
    print("Would run '%s'" % cmd)
    return

with open(log, "a") as outf:
    writelog("Running '%s' with output to %s\n" % (cmd, log), outf, sys.stdout)

with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0) as p, open(log, 'ab') as f:
    for line in p.stdout:
        writelog(line, f, sys.stdout.buffer)
        sys.stdout.flush()
        f.flush()
    ret = p.wait()
if ret:
    hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
    # No error report was written but the command failed so we should write one
    try:
        finalnumreports = len(os.listdir(errordir))
    except FileNotFoundError:
        finalnumreports = 0
    if finalnumreports == numreports:
        report.create(cmd, stepnum, log)
    finalret += 1
    errorlogs.add(log)

def runcmd(cmd, *args, **kwargs): if testmode: print("Running %s" % cmd) if "setup-config" not in cmd[0]: return try: subprocess.check_call(cmd, *args, **kwargs) except subprocess.CalledProcessError: print("ERROR: Command %s failed" % cmd)

bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname, 1) if bh_path: if jcfg: addentry("buildhistory-init", "Initialize buildhistory", "init") if args.phase == "init" and args.stepname == "buildhistory-init": if bh_path: runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch]) sys.exit(0)

def handle_stepnum(stepnum): shortdesc = utils.getconfigvar("shortname", ourconfig, args.target, stepnum) or "" desc = utils.getconfigvar("description", ourconfig, args.target, stepnum) or ""

# Add any layers specified
layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum)
if jcfg:
    if layers:
        addstepentry("add-layers", "Add layers", shortdesc, desc, str(layers), str(stepnum))
elif args.stepname == "add-layers":
    for layer in layers:
        bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, args.stepname)
    log_file_contents(args.builddir + "/conf/bblayers.conf", args.builddir, stepnum, args.stepname)

flush()

# Generate the configuration files needed for this step
if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
    if jcfg:
        addstepentry("write-config", "Write config", shortdesc, desc, None, str(stepnum))
    elif args.stepname == "write-config":
        runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])
        log_file_contents(args.builddir + "/conf/auto.conf", args.builddir, stepnum, args.stepname)

# Execute the targets for this configuration
targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
if targets:
    if jcfg:
        addstepentry("build-targets", "Build targets", shortdesc, desc, str(targets), str(stepnum))
    elif args.stepname == "build-targets":
        hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
        bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, args.stepname)

# Execute the sanity targets for this configuration
sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
if sanitytargets:
    if jcfg:
        addstepentry("test-targets", "QA targets", shortdesc, desc, str(sanitytargets), str(stepnum))
    elif args.stepname == "test-targets":
        hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
        bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, args.stepname)

# Run any extra commands specified
cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
if jcfg:
    if cmds:
        usepty = False
        if utils.getconfigvar("USEPTY", ourconfig, args.target, stepnum):
            usepty = True
        addstepentry("cmds", "Run cmds", shortdesc, desc, str(cmds), str(stepnum), usepty=usepty)
elif args.stepname == "cmds":
    for cmd in cmds:
        hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
        bitbakecmd(args.builddir, cmd, report, stepnum, args.stepname)

cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
if jcfg:
    if cmds:
        addstepentry("plain-cmds", "Run cmds", shortdesc, desc, str(cmds), str(stepnum))
elif args.stepname == "plain-cmds":
    for cmd in cmds:
        hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
        bitbakecmd(args.builddir, cmd, report, stepnum, args.stepname, oeenv=False)

if jcfg:
    if layers:
        addstepentry("remove-layers", "Remove layers", shortdesc, desc, str(layers), str(stepnum))
elif args.stepname == "remove-layers":
    # Remove any layers we added in a reverse order
    for layer in reversed(layers):
        bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, args.stepname)
    log_file_contents(args.builddir + "/conf/bblayers.conf", args.builddir, stepnum, args.stepname)

if not jcfg:
    sys.exit(finalret)

if jcfg: for stepnum in range(1, maxsteps + 1): handle_stepnum(stepnum) else: try: stepnum = int(args.phase) except ValueError: stepnum = None

if stepnum is not None:
    handle_stepnum(stepnum)

if jcfg: addentry("publish", "Publishing artefacts", "finish") elif args.phase == "finish" and args.stepname == "publish": if args.publish_dir: hp.printheader("Running publish artefacts") runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target]) sys.exit(0)

if jcfg: addentry("collect-results", "Collecting result files", "finish") elif args.phase == "finish" and args.stepname == "collect-results": if args.results_dir: hp.printheader("Running results collection") runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target]) runcmd([scriptsdir + "/summarize_top_output.py", args.results_dir, args.target]) runcmd([scriptsdir + "/archive_buildstats.py", args.builddir, args.results_dir, args.target]) sys.exit(0)

if jcfg: addentry("send-errors", "Sending error reports", "finish") elif args.phase == "finish" and args.stepname == "send-errors": if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum): hp.printheader("Sending any error reports") runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url]) sys.exit(0)

if jcfg: addentry("builddir-cleanup", "Cleaning up build directory", "finish") elif args.phase == "finish" and args.stepname == "builddir-cleanup": if args.builddir and os.path.exists(args.builddir): if os.path.exists("oe-init-build-env"): bitbakecmd(args.builddir, "bitbake -m", report, 99, args.stepname) runcmd(["mv", args.builddir, args.builddir + "-renamed"])

if args.json_outputfile: with open(args.json_outputfile, "w") as f: json.dump(jsonconfig, f, indent=4, sort_keys=True)

sys.exit(0)