#!/usr/bin/env python3
#
# Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.
#

import json
import os
import sys
import subprocess
import errno
import fnmatch
import glob
import fcntl

import utils

parser = utils.ArgParser(description='Runs configurations in json.conf.')

parser.add_argument('target',
                    help="The 'nightly' target the autobuilder is running")
parser.add_argument('builddir',
                    help="The target build directory to configure")
parser.add_argument('branchname',
                    help="The poky branch name the build is running on")
parser.add_argument('reponame',
                    help="The name of the repository the build is running on")
parser.add_argument('-s', '--sstateprefix',
                    default='',
                    help="The directory prefix to publish sstate into")
parser.add_argument('-b', '--buildappsrcrev',
                    default='',
                    help="A build appliance SRCREV to use")
parser.add_argument('-p', '--publish-dir',
                    action='store',
                    help="Where to publish artefacts to (optional)")
parser.add_argument('-r', '--results-dir',
                    action='store',
                    help="Where to publish test results to (optional)")
parser.add_argument('-u', '--build-url',
                    action='store',
                    help="URL back to this build (for the error reporting system)")
parser.add_argument('--build-type',
                    action='store',
                    default="quick",
                    help="the type of build being triggered (full or quick)")
parser.add_argument('-t', '--test',
                    action='store_true',
                    default=False,
                    help="Test mode - perform setup and dry-run of commands only")
parser.add_argument('-q', '--quietlogging',
                    action='store_true',
                    default=False,
                    help="Quiet mode - don't echo bitbake logs to stdout")
parser.add_argument('--workername',
                    action='store',
                    default=None,
                    help="the name of the worker the build is running on")


args = parser.parse_args()

scriptsdir = os.path.dirname(os.path.realpath(__file__))
os.environ["SCRIPTSDIR"] = scriptsdir
ourconfig = utils.loadconfig()
ourconfig["HELPERBUILDDIR"] = args.builddir
ourconfig["HELPERTARGET"] = args.target
ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "")
ourconfig["HELPERREPONAME"] = args.reponame
ourconfig["HELPERBRANCHNAME"] = args.branchname

hp = utils.HeaderPrinter()

testmode = args.test

# toolchain tests are run in system mode for x86, user mode for the other 
# arches due to speed
# toolchain tests only run on full builds
if args.build_type == "quick":
    ourconfig["HELPERSTMACHTARGS"] = "-a -t machine"
elif args.build_type == "full":
    if args.target == "qemux86" or args.target == "qemux86-64":
        ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-system"
    else:
        ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-user"

# Find out the number of steps this target has
maxsteps = 1
if args.target in ourconfig['overrides']:
    for v in ourconfig['overrides'][args.target]:
        if v.startswith("step"):
            n = int(v[4:])
            if n <= maxsteps:
                continue
            maxsteps = n

hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))

bttarball = None
if "buildtools" in ourconfig and args.workername:
    btcfg = utils.getconfig("buildtools", ourconfig)
    for entry in btcfg:
        if fnmatch.fnmatch(args.workername, entry):
            bttarball = btcfg[entry]
            break

btenv = None
if bttarball:
    btdir = os.path.abspath(args.builddir + "/../buildtools")
    if not os.path.exists(btdir):
        btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
        print("Extracting buildtools %s" % bttarball)
        btlock = btdlpath + ".lock"
        if not os.path.exists(os.path.dirname(btdlpath)):
            os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
        while True:
            try:
                with open(btlock, 'a+') as lf:
                    fileno = lf.fileno()
                    fcntl.flock(fileno, fcntl.LOCK_EX)
                    if not os.path.exists(btdlpath):
                        if bttarball.startswith("/"):
                            subprocess.check_call(["cp", bttarball, btdlpath])
                        else:
                            subprocess.check_call(["wget", "-O", btdlpath, bttarball])
                        os.chmod(btdlpath, 0o775)
                break
            except OSError:
                # We raced with someone else, try again
                pass
        subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
    btenv = glob.glob(btdir + "/environment-setup*")
    print("Using buildtools %s" % btenv)
    # We either parse or wrap all our execution calls, rock and a hard place :(
    with open(btenv[0], "r") as f:
        for line in f.readlines():
            if line.startswith("export "):
                line = line.strip().split(" ", 1)[1].split("=", 1)
                if "$PATH" in line[1]:
                    line[1] = line[1].replace("$PATH", os.environ["PATH"])
                if line[1].startswith(("'", '"')):
                    line[1] = line[1][1:-1]
                os.environ[line[0]] = line[1]

logconfig = args.builddir + "/../bitbake/contrib/autobuilderlog.json"
print("Using BB_LOGCONFIG=%s" % logconfig)
os.environ["BB_LOGCONFIG"] = logconfig

finalret = 0

def flush():
    sys.stdout.flush()
    sys.stderr.flush()

lognum = 0
def logname(path, stepnum, logsuffix):
    global lognum
    lognum += 1
    return path + "/command.log.%s%s" % (stepnum, logsuffix)

utils.mkdir(args.builddir)

revision = "unknown"
report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision)
errordir = utils.errorreportdir(args.builddir)
utils.mkdir(errordir)

errorlogs = set()

def bitbakecmd(builddir, cmd, report, stepnum, logsuffix, oeenv=True):
    global finalret
    flush()
    log = logname(builddir, stepnum, logsuffix)
    errordir = utils.errorreportdir(builddir)
    try:
        numreports = len(os.listdir(errordir))
    except FileNotFoundError:
        numreports = 0

    if oeenv:
        cmd = ". ./oe-init-build-env; %s" % cmd

    if testmode:
        print("Would run '%s'" % cmd)
        return

    print("Running '%s' with output to %s" % (cmd, log))
    flush()

    autoconf = builddir + "/conf/auto.conf"
    if os.path.exists(autoconf):
        with open(autoconf, "r") as inf, open(log, "a") as outf:
            outf.write("auto.conf settings:\n")
            for line in inf.readlines():
                outf.write(line)
            outf.write("\n")

    with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p, open(log, 'ab') as f:
        for line in p.stdout:
            if not args.quietlogging:
                sys.stdout.buffer.write(line)
                sys.stdout.flush()
            f.write(line)
            f.flush()
        ret = p.wait()
    if ret:
        hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
        # No error report was written but the command failed so we should write one
        try:
            finalnumreports = len(os.listdir(errordir))
        except FileNotFoundError:
            finalnumreports = 0
        if finalnumreports == numreports:
            report.create(cmd, stepnum, log)
        finalret += 1
        errorlogs.add(log)

def runcmd(cmd, *args, **kwargs):
    if testmode:
        print("Running %s" % cmd)
        if "setup-config" not in cmd[0]:
            return
    try:
        subprocess.check_call(cmd, *args, **kwargs)
    except subprocess.CalledProcessError:
        print("ERROR: Command %s failed" % cmd)

bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname, 1)
if bh_path:
    runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])

for stepnum in range(1, maxsteps + 1):
    # Add any layers specified
    layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum)
    for layer in layers:
        bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, 'a')

    flush()
    # Generate the configuration files needed for this step
    if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
        runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])

    # Execute the targets for this configuration
    targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
    if targets:
        hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
        bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, 'b')

    # Execute the sanity targets for this configuration
    sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
    if sanitytargets:
        hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
        bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, 'c')

    # Run any extra commands specified
    cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
    for cmd in cmds:
        hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
        bitbakecmd(args.builddir, cmd, report, stepnum, 'd')
    cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
    for cmd in cmds:
        hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
        bitbakecmd(args.builddir, cmd, report, stepnum, 'd', oeenv=False)

    # Remove any layers we added in a reverse order
    for layer in reversed(layers):
        bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, 'a')

if args.publish_dir:
    hp.printheader("Running publish artefacts")
    runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target])

if args.results_dir:
    hp.printheader("Running results collection")
    runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])

if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum):
    hp.printheader("Sending any error reports")
    runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url])

if args.builddir and os.path.exists(args.builddir):
    # Clean up our build directory if things were successful and we're not publishing anything
    # (keep published builds around for longer just in case we need them)
    if not finalret and not args.publish_dir:
        runcmd([scriptsdir + "/../janitor/clobberdir", args.builddir])
    else:
        # Rename any completed build directory so that other builds can't reference paths within it
        runcmd(["mv", args.builddir, args.builddir + "-renamed"])

if finalret:
    hp.printheader("There were %s failures" % finalret)
    hp.printheader("Failures in logfiles: %s" % " ".join(errorlogs))
    sys.exit(1)

sys.exit(0)