diff --git a/builders.py b/builders.py index 294200f..f6d6fbc 100644 --- a/builders.py +++ b/builders.py @@ -2,10 +2,10 @@ from buildbot.plugins import * from yoctoabb import config from yoctoabb.steps.writelayerinfo import WriteLayerInfo +from yoctoabb.steps.runconfig import get_publish_dest, get_publish_resultdir, get_publish_name, get_runconfig_legacy_step from yoctoabb.steps.observer import RunConfigLogObserver from twisted.python import log -from datetime import datetime import os import json @@ -13,7 +13,6 @@ import random builders = [] -maxsteps = 9 # Environment to pass into the workers, e.g. to load further local configuration # fragments @@ -21,95 +20,6 @@ extra_env = {} if os.environ.get('ABHELPER_JSON'): extra_env['ABHELPER_JSON'] = os.environ['ABHELPER_JSON'] -@util.renderer -def get_sstate_release_number(props): - """ - Uses the values submitted to the scheduler to determine the major number - of the release for the purposes of publishing per-major release - shared-state artefacts. - """ - release_number = props.getProperty("yocto_number") - if not release_number: - return "" - release_components = release_number.split('.', 3) - return '.'.join(release_components).strip('.') - - -def get_publish_internal(props): - """ - Calculate the location to which artefacts should be published and store it - as a property for use by other workers. - """ - # Cache the value in the publish_detination property - dest = props.getProperty("publish_destination", "") - if dest: - return dest - - if props.getProperty("is_release", False): - milestone = props.getProperty("milestone_number", "") - rc_number = props.getProperty("rc_number", "") - snapshot = "" - if milestone: - snapshot += "_" + milestone - if rc_number: - snapshot += "." + rc_number - - rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot - dest = os.path.join(config.publish_dest, "releases", rel_name) - else: - dest_base = os.path.join(config.publish_dest, 'non-release', - datetime.now().strftime("%Y%m%d")) - - # We want to make sure that we aren't writing artefacts to a publish - # directory which already exists, therefore we keep a list of used - # publish paths to prevent re-use. We store that in a JSON file. - useddests = {} - # NOTE: we make a strong assumption here that this code lives in a - # directory which is an immediate child of the buildbot master's - # working directory. - basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "..") - persist = os.path.join(basedir, "pub_locations.json") - if os.path.exists(persist): - with open(persist) as f: - useddests = json.load(f) - - rev = useddests.get(dest_base, "") - if rev: # incremenent and use - rev = int(rev) + 1 - else: # use the base path and store rev 0 - rev = 1 - dest = "%s-%s" % (dest_base, rev) - # need to update the used destinations - useddests[dest_base] = rev - # save the info, overwriting the file if it exists - with open(persist, 'w') as out: - json.dump(useddests, out) - - # set the destination as a property to be inherited by workers, so that - # all workers in a triggered set publish to the same location - props.setProperty("publish_destination", dest, - "get_publish_dest") - return dest - -@util.renderer -def get_publish_dest(props): - deploy = props.getProperty("deploy_artefacts", False) - if not deploy: - return "" - return get_publish_internal(props) - -@util.renderer -def get_publish_resultdir(props): - return get_publish_internal(props) + "/testresults" - -@util.renderer -def get_publish_name(props): - dest = get_publish_internal(props) - if dest: - return os.path.basename(dest) - return dest - @util.renderer def ensure_props_set(props): """ @@ -126,13 +36,6 @@ def ensure_props_set(props): "publish_destination": props.getProperty("publish_destination", "") } -def get_buildlogs(maxsteps): - logfiles = {} - for i in range(1, maxsteps): - for j in ['a', 'b', 'c', 'd']: - logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j) - return logfiles - def create_builder_factory(): f = util.BuildFactory() @@ -173,25 +76,9 @@ def create_builder_factory(): haltOnFailure=True, name='Set build branch')) - f.addStep(RunConfigLogObserver( - command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"), - util.Property("buildername"), - util.Interpolate("%(prop:builddir)s/build/build"), - util.Interpolate("%(prop:branch_poky)s"), - util.Interpolate("%(prop:repo_poky)s"), - "--sstateprefix", get_sstate_release_number, - "--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"), - "--build-type", util.Interpolate("%(prop:build_type)s"), - "--workername", util.Interpolate("%(prop:workername)s"), - "--publish-dir", get_publish_dest, - "--build-url", util.URLForBuild, - "--results-dir", get_publish_resultdir, - "--quietlogging"], - name="run-config", - logfiles=get_buildlogs(maxsteps), - lazylogfiles=True, - maxsteps=maxsteps, - timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs + + f.addStep(get_runconfig_legacy_step(posttrigger=False)) + return f def nextWorker(bldr, workers, buildrequest): @@ -294,25 +181,7 @@ def create_parent_builder_factory(buildername, waitname): name='Set build branch')) # run-config - factory.addStep(RunConfigLogObserver( - command=[ - util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"), - util.Property("buildername"), - util.Interpolate("%(prop:builddir)s/build/build"), - util.Interpolate("%(prop:branch_poky)s"), - util.Interpolate("%(prop:repo_poky)s"), - "--sstateprefix", get_sstate_release_number, - "--publish-dir", get_publish_dest, - "--build-type", util.Interpolate("%(prop:build_type)s"), - "--workername", util.Interpolate("%(prop:workername)s"), - "--build-url", util.URLForBuild, - "--results-dir", get_publish_resultdir, - "--quietlogging"], - name="run-config", - logfiles=get_buildlogs(maxsteps), - lazylogfiles=True, - maxsteps=maxsteps, - timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs + factory.addStep(get_runconfig_legacy_step(posttrigger=False)) # trigger the buildsets contained in the nightly set def get_props_set(): @@ -342,26 +211,7 @@ def create_parent_builder_factory(buildername, waitname): waitForFinish=True, set_properties=get_props_set())) - factory.addStep(RunConfigLogObserver( - command=[ - util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"), - util.Interpolate("%(prop:buildername)s-posttrigger"), - util.Interpolate("%(prop:builddir)s/build/build"), - util.Interpolate("%(prop:branch_poky)s"), - util.Interpolate("%(prop:repo_poky)s"), - "--sstateprefix", get_sstate_release_number, - "--publish-dir", get_publish_dest, - "--build-type", util.Interpolate("%(prop:build_type)s"), - "--workername", util.Interpolate("%(prop:workername)s"), - "--build-url", util.URLForBuild, - "--results-dir", get_publish_resultdir, - "--quietlogging"], - name="run-config", - logfiles=get_buildlogs(maxsteps), - lazylogfiles=True, - maxsteps=maxsteps, - timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs - + factory.addStep(get_runconfig_legacy_step(posttrigger=True)) factory.addStep(steps.ShellCommand( command=[ diff --git a/steps/runconfig.py b/steps/runconfig.py new file mode 100644 index 0000000..074e618 --- /dev/null +++ b/steps/runconfig.py @@ -0,0 +1,141 @@ +from twisted.internet import defer +from buildbot.plugins import steps, util +from buildbot.process import buildstep +from buildbot.process.results import Results, SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY + +from yoctoabb.steps.observer import RunConfigLogObserver + +import json +import datetime +import os +import sys + +from yoctoabb import config + +maxsteps = 9 + +def get_publish_internal(props): + """ + Calculate the location to which artefacts should be published and store it + as a property for use by other workers. + """ + # Cache the value in the publish_detination property + dest = props.getProperty("publish_destination", "") + if dest: + return dest + + if props.getProperty("is_release", False): + milestone = props.getProperty("milestone_number", "") + rc_number = props.getProperty("rc_number", "") + snapshot = "" + if milestone: + snapshot += "_" + milestone + if rc_number: + snapshot += "." + rc_number + + rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot + dest = os.path.join(config.publish_dest, "releases", rel_name) + else: + dest_base = os.path.join(config.publish_dest, 'non-release', + datetime.now().strftime("%Y%m%d")) + + # We want to make sure that we aren't writing artefacts to a publish + # directory which already exists, therefore we keep a list of used + # publish paths to prevent re-use. We store that in a JSON file. + useddests = {} + # NOTE: we make a strong assumption here that this code lives in a + # directory which is an immediate child of the buildbot master's + # working directory. + basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..") + persist = os.path.join(basedir, "pub_locations.json") + if os.path.exists(persist): + with open(persist) as f: + useddests = json.load(f) + + rev = useddests.get(dest_base, "") + if rev: # incremenent and use + rev = int(rev) + 1 + else: # use the base path and store rev 0 + rev = 1 + dest = "%s-%s" % (dest_base, rev) + # need to update the used destinations + useddests[dest_base] = rev + # save the info, overwriting the file if it exists + with open(persist, 'w') as out: + json.dump(useddests, out) + + # set the destination as a property to be inherited by workers, so that + # all workers in a triggered set publish to the same location + props.setProperty("publish_destination", dest, + "get_publish_dest") + return dest + +@util.renderer +def get_publish_dest(props): + deploy = props.getProperty("deploy_artefacts", False) + if not deploy: + return "" + return get_publish_internal(props) + +@util.renderer +def get_publish_resultdir(props): + return get_publish_internal(props) + "/testresults" + +@util.renderer +def get_publish_name(props): + dest = get_publish_internal(props) + if dest: + return os.path.basename(dest) + return dest + +@util.renderer +def get_sstate_release_number(props): + """ + Uses the values submitted to the scheduler to determine the major number + of the release for the purposes of publishing per-major release + shared-state artefacts. + """ + release_number = props.getProperty("yocto_number") + if not release_number: + return "" + release_components = release_number.split('.', 3) + return '.'.join(release_components).strip('.') + +def get_runconfig_command(posttrigger=False): + runconfig_command = [util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config")] + if posttrigger: + runconfig_command.append(util.Interpolate("%(prop:buildername)s-posttrigger")) + else: + runconfig_command.append(util.Property("buildername")) + runconfig_command.extend([ + util.Interpolate("%(prop:builddir)s/build/build"), + util.Interpolate("%(prop:branch_poky)s"), + util.Interpolate("%(prop:repo_poky)s"), + "--sstateprefix", get_sstate_release_number, + "--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"), + "--publish-dir", get_publish_dest, + "--build-type", util.Interpolate("%(prop:build_type)s"), + "--workername", util.Interpolate("%(prop:workername)s"), + "--build-url", util.URLForBuild, + "--results-dir", get_publish_resultdir, + "--quietlogging"]) + return runconfig_command + +def get_buildlogs(maxsteps): + logfiles = {} + for i in range(1, maxsteps): + for j in ['a', 'b', 'c', 'd']: + logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j) + return logfiles + +def get_runconfig_legacy_step(posttrigger): + step = RunConfigLogObserver( + command=get_runconfig_command(posttrigger), + name="run-config", + logfiles=get_buildlogs(maxsteps), + lazylogfiles=True, + maxsteps=maxsteps, + timeout=16200) # default of 1200s/20min is too short, use 4.5hrs + return step +