builders: Move run-config handling to its own module

Also abstract the calls into a function to allow for easier further
development.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

builders.py

@@ -2,10 +2,10 @@ from buildbot.plugins import *
 
 from yoctoabb import config
 from yoctoabb.steps.writelayerinfo import WriteLayerInfo
+from yoctoabb.steps.runconfig import get_publish_dest, get_publish_resultdir, get_publish_name, get_runconfig_legacy_step
 from yoctoabb.steps.observer import RunConfigLogObserver
 
 from twisted.python import log
-from datetime import datetime
 
 import os
 import json
@@ -13,7 +13,6 @@ import random
 
 
 builders = []
-maxsteps = 9
 
 # Environment to pass into the workers, e.g. to load further local configuration
 # fragments
@@ -21,95 +20,6 @@ extra_env = {}
 if os.environ.get('ABHELPER_JSON'):
     extra_env['ABHELPER_JSON'] = os.environ['ABHELPER_JSON']
 
-@util.renderer
-def get_sstate_release_number(props):
-    """
-    Uses the values submitted to the scheduler to determine the major number
-    of the release for the purposes of publishing per-major release
-    shared-state artefacts.
-    """
-    release_number = props.getProperty("yocto_number")
-    if not release_number:
-        return ""
-    release_components = release_number.split('.', 3)
-    return '.'.join(release_components).strip('.')
-
-
-def get_publish_internal(props):
-    """
-    Calculate the location to which artefacts should be published and store it
-    as a property for use by other workers.
-    """
-    # Cache the value in the publish_detination property
-    dest = props.getProperty("publish_destination", "")
-    if dest:
-        return dest
-
-    if props.getProperty("is_release", False):
-        milestone = props.getProperty("milestone_number", "")
-        rc_number = props.getProperty("rc_number", "")
-        snapshot = ""
-        if milestone:
-            snapshot += "_" + milestone
-        if rc_number:
-            snapshot += "." + rc_number
-
-        rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
-        dest = os.path.join(config.publish_dest, "releases", rel_name)
-    else:
-        dest_base = os.path.join(config.publish_dest, 'non-release',
-                                 datetime.now().strftime("%Y%m%d"))
-
-        # We want to make sure that we aren't writing artefacts to a publish
-        # directory which already exists, therefore we keep a list of used
-        # publish paths to prevent re-use. We store that in a JSON file.
-        useddests = {}
-        # NOTE: we make a strong assumption here that this code lives in a
-        # directory which is an immediate child of the buildbot master's
-        # working directory.
-        basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                               "..")
-        persist = os.path.join(basedir, "pub_locations.json")
-        if os.path.exists(persist):
-            with open(persist) as f:
-                useddests = json.load(f)
-
-        rev = useddests.get(dest_base, "")
-        if rev:  # incremenent and use
-            rev = int(rev) + 1
-        else:  # use the base path and store rev 0
-            rev = 1
-        dest = "%s-%s" % (dest_base, rev)
-        # need to update the used destinations
-        useddests[dest_base] = rev
-        # save the info, overwriting the file if it exists
-        with open(persist, 'w') as out:
-            json.dump(useddests, out)
-
-    # set the destination as a property to be inherited by workers, so that
-    # all workers in a triggered set publish to the same location
-    props.setProperty("publish_destination", dest,
-                          "get_publish_dest")
-    return dest
-
-@util.renderer
-def get_publish_dest(props):
-    deploy = props.getProperty("deploy_artefacts", False)
-    if not deploy:
-        return ""
-    return get_publish_internal(props)
-
-@util.renderer
-def get_publish_resultdir(props):
-    return get_publish_internal(props) + "/testresults"
-
-@util.renderer
-def get_publish_name(props):
-    dest = get_publish_internal(props)
-    if dest:
-        return os.path.basename(dest)
-    return dest
-
 @util.renderer
 def ensure_props_set(props):
     """
@@ -126,13 +36,6 @@ def ensure_props_set(props):
         "publish_destination": props.getProperty("publish_destination", "")
     }
 
-def get_buildlogs(maxsteps):
-    logfiles = {}
-    for i in range(1, maxsteps):
-        for j in ['a', 'b', 'c', 'd']:
-            logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j)
-    return logfiles
-
 def create_builder_factory():
     f = util.BuildFactory()
 
@@ -173,25 +76,9 @@ def create_builder_factory():
                                            haltOnFailure=True,
                                            name='Set build branch'))
 
-    f.addStep(RunConfigLogObserver(
+
-        command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
+    f.addStep(get_runconfig_legacy_step(posttrigger=False))
-                 util.Property("buildername"),
+
-                 util.Interpolate("%(prop:builddir)s/build/build"),
-                 util.Interpolate("%(prop:branch_poky)s"),
-                 util.Interpolate("%(prop:repo_poky)s"),
-                 "--sstateprefix", get_sstate_release_number,
-                 "--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"),
-                 "--build-type", util.Interpolate("%(prop:build_type)s"),
-                 "--workername", util.Interpolate("%(prop:workername)s"),
-                 "--publish-dir", get_publish_dest,
-                 "--build-url", util.URLForBuild,
-                 "--results-dir", get_publish_resultdir,
-                 "--quietlogging"],
-        name="run-config",
-        logfiles=get_buildlogs(maxsteps),
-        lazylogfiles=True,
-        maxsteps=maxsteps,
-        timeout=16200))  # default of 1200s/20min is too short, use 4.5hrs
     return f
 
 def nextWorker(bldr, workers, buildrequest):
@@ -294,25 +181,7 @@ def create_parent_builder_factory(buildername, waitname):
                                                  name='Set build branch'))
 
     # run-config
-    factory.addStep(RunConfigLogObserver(
+    factory.addStep(get_runconfig_legacy_step(posttrigger=False))
-        command=[
-            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
-            util.Property("buildername"),
-            util.Interpolate("%(prop:builddir)s/build/build"),
-            util.Interpolate("%(prop:branch_poky)s"),
-            util.Interpolate("%(prop:repo_poky)s"),
-            "--sstateprefix", get_sstate_release_number,
-            "--publish-dir", get_publish_dest,
-            "--build-type", util.Interpolate("%(prop:build_type)s"),
-            "--workername", util.Interpolate("%(prop:workername)s"),
-            "--build-url", util.URLForBuild,
-            "--results-dir", get_publish_resultdir,
-            "--quietlogging"],
-        name="run-config",
-        logfiles=get_buildlogs(maxsteps),
-        lazylogfiles=True,
-        maxsteps=maxsteps,
-        timeout=16200))  # default of 1200s/20min is too short, use 4.5hrs
 
     # trigger the buildsets contained in the nightly set
     def get_props_set():
@@ -342,26 +211,7 @@ def create_parent_builder_factory(buildername, waitname):
                                   waitForFinish=True,
                                   set_properties=get_props_set()))
 
-    factory.addStep(RunConfigLogObserver(
+    factory.addStep(get_runconfig_legacy_step(posttrigger=True))
-        command=[
-            util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
-            util.Interpolate("%(prop:buildername)s-posttrigger"),
-            util.Interpolate("%(prop:builddir)s/build/build"),
-            util.Interpolate("%(prop:branch_poky)s"),
-            util.Interpolate("%(prop:repo_poky)s"),
-            "--sstateprefix", get_sstate_release_number,
-            "--publish-dir", get_publish_dest,
-            "--build-type", util.Interpolate("%(prop:build_type)s"),
-            "--workername", util.Interpolate("%(prop:workername)s"),
-            "--build-url", util.URLForBuild,
-            "--results-dir", get_publish_resultdir,
-            "--quietlogging"],
-        name="run-config",
-        logfiles=get_buildlogs(maxsteps),
-        lazylogfiles=True,
-        maxsteps=maxsteps,
-        timeout=16200))  # default of 1200s/20min is too short, use 4.5hrs
-
 
     factory.addStep(steps.ShellCommand(
         command=[

steps/runconfig.py

@@ -0,0 +1,141 @@
+from twisted.internet import defer
+from buildbot.plugins import steps, util
+from buildbot.process import buildstep
+from buildbot.process.results import Results, SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY
+
+from yoctoabb.steps.observer import RunConfigLogObserver
+
+import json
+import datetime
+import os
+import sys
+
+from yoctoabb import config
+
+maxsteps = 9
+
+def get_publish_internal(props):
+    """
+    Calculate the location to which artefacts should be published and store it
+    as a property for use by other workers.
+    """
+    # Cache the value in the publish_detination property
+    dest = props.getProperty("publish_destination", "")
+    if dest:
+        return dest
+
+    if props.getProperty("is_release", False):
+        milestone = props.getProperty("milestone_number", "")
+        rc_number = props.getProperty("rc_number", "")
+        snapshot = ""
+        if milestone:
+            snapshot += "_" + milestone
+        if rc_number:
+            snapshot += "." + rc_number
+
+        rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
+        dest = os.path.join(config.publish_dest, "releases", rel_name)
+    else:
+        dest_base = os.path.join(config.publish_dest, 'non-release',
+                                 datetime.now().strftime("%Y%m%d"))
+
+        # We want to make sure that we aren't writing artefacts to a publish
+        # directory which already exists, therefore we keep a list of used
+        # publish paths to prevent re-use. We store that in a JSON file.
+        useddests = {}
+        # NOTE: we make a strong assumption here that this code lives in a
+        # directory which is an immediate child of the buildbot master's
+        # working directory.
+        basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                               "..")
+        persist = os.path.join(basedir, "pub_locations.json")
+        if os.path.exists(persist):
+            with open(persist) as f:
+                useddests = json.load(f)
+
+        rev = useddests.get(dest_base, "")
+        if rev:  # incremenent and use
+            rev = int(rev) + 1
+        else:  # use the base path and store rev 0
+            rev = 1
+        dest = "%s-%s" % (dest_base, rev)
+        # need to update the used destinations
+        useddests[dest_base] = rev
+        # save the info, overwriting the file if it exists
+        with open(persist, 'w') as out:
+            json.dump(useddests, out)
+
+    # set the destination as a property to be inherited by workers, so that
+    # all workers in a triggered set publish to the same location
+    props.setProperty("publish_destination", dest,
+                          "get_publish_dest")
+    return dest
+
+@util.renderer
+def get_publish_dest(props):
+    deploy = props.getProperty("deploy_artefacts", False)
+    if not deploy:
+        return ""
+    return get_publish_internal(props)
+
+@util.renderer
+def get_publish_resultdir(props):
+    return get_publish_internal(props) + "/testresults"
+
+@util.renderer
+def get_publish_name(props):
+    dest = get_publish_internal(props)
+    if dest:
+        return os.path.basename(dest)
+    return dest
+
+@util.renderer
+def get_sstate_release_number(props):
+    """
+    Uses the values submitted to the scheduler to determine the major number
+    of the release for the purposes of publishing per-major release
+    shared-state artefacts.
+    """
+    release_number = props.getProperty("yocto_number")
+    if not release_number:
+        return ""
+    release_components = release_number.split('.', 3)
+    return '.'.join(release_components).strip('.')
+
+def get_runconfig_command(posttrigger=False):
+    runconfig_command = [util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config")]
+    if posttrigger:
+        runconfig_command.append(util.Interpolate("%(prop:buildername)s-posttrigger"))
+    else:
+        runconfig_command.append(util.Property("buildername"))
+    runconfig_command.extend([ 
+        util.Interpolate("%(prop:builddir)s/build/build"),
+        util.Interpolate("%(prop:branch_poky)s"),
+        util.Interpolate("%(prop:repo_poky)s"),
+        "--sstateprefix", get_sstate_release_number,
+        "--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"),
+        "--publish-dir", get_publish_dest,
+        "--build-type", util.Interpolate("%(prop:build_type)s"),
+        "--workername", util.Interpolate("%(prop:workername)s"),
+        "--build-url", util.URLForBuild,
+        "--results-dir", get_publish_resultdir,
+        "--quietlogging"])
+    return runconfig_command
+
+def get_buildlogs(maxsteps):
+    logfiles = {}
+    for i in range(1, maxsteps):
+        for j in ['a', 'b', 'c', 'd']:
+            logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j)
+    return logfiles
+
+def get_runconfig_legacy_step(posttrigger):
+    step = RunConfigLogObserver(
+        command=get_runconfig_command(posttrigger),
+        name="run-config",
+        logfiles=get_buildlogs(maxsteps),
+        lazylogfiles=True,
+        maxsteps=maxsteps,
+        timeout=16200)  # default of 1200s/20min is too short, use 4.5hrs
+    return step
+