mirror of
git://git.yoctoproject.org/yocto-autobuilder2.git
synced 2025-07-19 03:49:11 +02:00
25d5a7d0ae
Rather than running all the work in a single run-config command, split this into multiple steps. This allows for separate result code reporting, timestamps for indvidual steps and is generally much cleaner. The naming and so on will need further work but this is a start. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
207 lines
7.6 KiB
Python
207 lines
7.6 KiB
Python
from twisted.internet import defer
|
|
from buildbot.plugins import steps, util
|
|
from buildbot.process import buildstep, logobserver
|
|
from buildbot.process.results import Results, SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY
|
|
from buildbot.steps import shell
|
|
|
|
from yoctoabb.steps.observer import RunConfigLogObserver
|
|
|
|
import json
|
|
import datetime
|
|
import os
|
|
import sys
|
|
|
|
from yoctoabb import config
|
|
|
|
maxsteps = 9
|
|
|
|
def get_publish_internal(props):
|
|
"""
|
|
Calculate the location to which artefacts should be published and store it
|
|
as a property for use by other workers.
|
|
"""
|
|
# Cache the value in the publish_detination property
|
|
dest = props.getProperty("publish_destination", "")
|
|
if dest:
|
|
return dest
|
|
|
|
if props.getProperty("is_release", False):
|
|
milestone = props.getProperty("milestone_number", "")
|
|
rc_number = props.getProperty("rc_number", "")
|
|
snapshot = ""
|
|
if milestone:
|
|
snapshot += "_" + milestone
|
|
if rc_number:
|
|
snapshot += "." + rc_number
|
|
|
|
rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
|
|
dest = os.path.join(config.publish_dest, "releases", rel_name)
|
|
else:
|
|
dest_base = os.path.join(config.publish_dest, 'non-release',
|
|
datetime.datetime.now().strftime("%Y%m%d"))
|
|
|
|
# We want to make sure that we aren't writing artefacts to a publish
|
|
# directory which already exists, therefore we keep a list of used
|
|
# publish paths to prevent re-use. We store that in a JSON file.
|
|
useddests = {}
|
|
# NOTE: we make a strong assumption here that this code lives in a
|
|
# directory which is an immediate child of the buildbot master's
|
|
# working directory.
|
|
basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
"..")
|
|
persist = os.path.join(basedir, "pub_locations.json")
|
|
if os.path.exists(persist):
|
|
with open(persist) as f:
|
|
useddests = json.load(f)
|
|
|
|
rev = useddests.get(dest_base, "")
|
|
if rev: # incremenent and use
|
|
rev = int(rev) + 1
|
|
else: # use the base path and store rev 0
|
|
rev = 1
|
|
dest = "%s-%s" % (dest_base, rev)
|
|
# need to update the used destinations
|
|
useddests[dest_base] = rev
|
|
# save the info, overwriting the file if it exists
|
|
with open(persist, 'w') as out:
|
|
json.dump(useddests, out)
|
|
|
|
# set the destination as a property to be inherited by workers, so that
|
|
# all workers in a triggered set publish to the same location
|
|
props.setProperty("publish_destination", dest,
|
|
"get_publish_dest")
|
|
return dest
|
|
|
|
@util.renderer
|
|
def get_publish_dest(props):
|
|
deploy = props.getProperty("deploy_artefacts", False)
|
|
if not deploy:
|
|
return ""
|
|
return get_publish_internal(props)
|
|
|
|
@util.renderer
|
|
def get_publish_resultdir(props):
|
|
return get_publish_internal(props) + "/testresults"
|
|
|
|
@util.renderer
|
|
def get_publish_name(props):
|
|
dest = get_publish_internal(props)
|
|
if dest:
|
|
return os.path.basename(dest)
|
|
return dest
|
|
|
|
@util.renderer
|
|
def get_sstate_release_number(props):
|
|
"""
|
|
Uses the values submitted to the scheduler to determine the major number
|
|
of the release for the purposes of publishing per-major release
|
|
shared-state artefacts.
|
|
"""
|
|
release_number = props.getProperty("yocto_number")
|
|
if not release_number:
|
|
return ""
|
|
release_components = release_number.split('.', 3)
|
|
return '.'.join(release_components).strip('.')
|
|
|
|
def get_runconfig_command(posttrigger=False):
|
|
runconfig_command = [util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config")]
|
|
if posttrigger:
|
|
runconfig_command.append(util.Interpolate("%(prop:buildername)s-posttrigger"))
|
|
else:
|
|
runconfig_command.append(util.Property("buildername"))
|
|
runconfig_command.extend([
|
|
util.Interpolate("%(prop:builddir)s/build/build"),
|
|
util.Interpolate("%(prop:branch_poky)s"),
|
|
util.Interpolate("%(prop:repo_poky)s"),
|
|
"--sstateprefix", get_sstate_release_number,
|
|
"--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"),
|
|
"--publish-dir", get_publish_dest,
|
|
"--build-type", util.Interpolate("%(prop:build_type)s"),
|
|
"--workername", util.Interpolate("%(prop:workername)s"),
|
|
"--build-url", util.URLForBuild,
|
|
"--results-dir", get_publish_resultdir,
|
|
"--quietlogging"])
|
|
return runconfig_command
|
|
|
|
def get_buildlogs(maxsteps):
|
|
logfiles = {}
|
|
for i in range(1, maxsteps):
|
|
for j in ['a', 'b', 'c', 'd']:
|
|
logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j)
|
|
return logfiles
|
|
|
|
def get_runconfig_legacy_step(posttrigger):
|
|
step = RunConfigLogObserver(
|
|
command=get_runconfig_command(posttrigger),
|
|
name="run-config",
|
|
logfiles=get_buildlogs(maxsteps),
|
|
lazylogfiles=True,
|
|
maxsteps=maxsteps,
|
|
timeout=16200) # default of 1200s/20min is too short, use 4.5hrs
|
|
return step
|
|
|
|
def get_runconfig_step(name, stepname, phase, description, posttrigger):
|
|
step = RunConfigLogObserver(
|
|
command=get_runconfig_command(posttrigger) + ['--stepname', stepname, '--phase', phase],
|
|
name=name,
|
|
description=description,
|
|
logfiles=get_buildlogs(maxsteps),
|
|
lazylogfiles=True,
|
|
maxsteps=maxsteps,
|
|
timeout=16200) # default of 1200s/20min is too short, use 4.5hrs
|
|
return step
|
|
|
|
class RunConfigCheckSteps(shell.ShellCommand):
|
|
name = 'Check run-config steps to use'
|
|
descriptionDone = ['Checked which run-config approach to use']
|
|
haltOnFailure = False
|
|
flunkOnFailure = True
|
|
jsonFileName = util.Interpolate("%(prop:builddir)s/runconfig.json")
|
|
logfiles = {'json': jsonFileName}
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.posttrigger = kwargs.pop("posttrigger")
|
|
self.command = get_runconfig_command(self.posttrigger)
|
|
self.command.append("--json-outputfile")
|
|
self.command.append(self.jsonFileName)
|
|
super().__init__(*args, **kwargs)
|
|
|
|
def start(self):
|
|
self.log_observer_json = logobserver.BufferLogObserver()
|
|
self.addLogObserver('json', self.log_observer_json)
|
|
return super().start()
|
|
|
|
def evaluateCommand(self, cmd):
|
|
# If the command fails, fall back to old style run-config execution
|
|
rc = super().evaluateCommand(cmd)
|
|
jsonconfig = self.getProperty("runconfig-json", None)
|
|
if rc == FAILURE or not jsonconfig:
|
|
steps = [get_runconfig_legacy_step(self.posttrigger)]
|
|
else:
|
|
steps = []
|
|
for s in jsonconfig:
|
|
name = "run-config-" + s['name'] + "-" + s['phase']
|
|
steps.append(get_runconfig_step(name, s['name'], s['phase'], name, self.posttrigger))
|
|
self.build.addStepsAfterCurrentStep(steps)
|
|
return SUCCESS
|
|
|
|
def commandComplete(self, cmd):
|
|
super().commandComplete(cmd)
|
|
logLines = self.log_observer_json.getStdout()
|
|
json_text = ''.join([line for line in logLines.splitlines()])
|
|
if not json_text:
|
|
return
|
|
try:
|
|
jsonconfig = json.loads(json_text)
|
|
self.setProperty("runconfig-json", jsonconfig)
|
|
except Exception as ex:
|
|
self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
|
|
|
|
@defer.inlineCallbacks
|
|
def _addToLog(self, logName, message):
|
|
try:
|
|
log = self.getLog(logName)
|
|
except KeyError:
|
|
log = yield self.addLog(logName)
|
|
log.addStdout(message)
|