From 25d5a7d0aec3d24c8247c93b2e4ab704445dd75a Mon Sep 17 00:00:00 2001
From: Richard Purdie <richard.purdie@linuxfoundation.org>
Date: Wed, 11 Nov 2020 18:01:23 +0000
Subject: [PATCH] builders/runconfig: Allow execution of run-config steps
 individually

Rather than running all the work in a single run-config command, split this into
multiple steps. This allows for separate result code reporting, timestamps
for indvidual steps and is generally much cleaner. The naming and so on will need
further work but this is a start.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 builders.py        |  8 +++---
 steps/runconfig.py | 67 +++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/builders.py b/builders.py
index f6d6fbc..ed0ddcd 100644
--- a/builders.py
+++ b/builders.py
@@ -2,7 +2,7 @@ from buildbot.plugins import *
 
 from yoctoabb import config
 from yoctoabb.steps.writelayerinfo import WriteLayerInfo
-from yoctoabb.steps.runconfig import get_publish_dest, get_publish_resultdir, get_publish_name, get_runconfig_legacy_step
+from yoctoabb.steps.runconfig import get_publish_dest, get_publish_resultdir, get_publish_name, RunConfigCheckSteps
 from yoctoabb.steps.observer import RunConfigLogObserver
 
 from twisted.python import log
@@ -77,7 +77,7 @@ def create_builder_factory():
                                            name='Set build branch'))
 
 
-    f.addStep(get_runconfig_legacy_step(posttrigger=False))
+    f.addStep(RunConfigCheckSteps(posttrigger=False))
 
     return f
 
@@ -181,7 +181,7 @@ def create_parent_builder_factory(buildername, waitname):
                                                  name='Set build branch'))
 
     # run-config
-    factory.addStep(get_runconfig_legacy_step(posttrigger=False))
+    factory.addStep(RunConfigCheckSteps(posttrigger=False))
 
     # trigger the buildsets contained in the nightly set
     def get_props_set():
@@ -211,7 +211,7 @@ def create_parent_builder_factory(buildername, waitname):
                                   waitForFinish=True,
                                   set_properties=get_props_set()))
 
-    factory.addStep(get_runconfig_legacy_step(posttrigger=True))
+    factory.addStep(RunConfigCheckSteps(posttrigger=True))
 
     factory.addStep(steps.ShellCommand(
         command=[
diff --git a/steps/runconfig.py b/steps/runconfig.py
index e1ce9f0..294a977 100644
--- a/steps/runconfig.py
+++ b/steps/runconfig.py
@@ -1,7 +1,8 @@
 from twisted.internet import defer
 from buildbot.plugins import steps, util
-from buildbot.process import buildstep
+from buildbot.process import buildstep, logobserver
 from buildbot.process.results import Results, SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY
+from buildbot.steps import shell
 
 from yoctoabb.steps.observer import RunConfigLogObserver
 
@@ -139,3 +140,67 @@ def get_runconfig_legacy_step(posttrigger):
         timeout=16200)  # default of 1200s/20min is too short, use 4.5hrs
     return step
 
+def get_runconfig_step(name, stepname, phase, description, posttrigger):
+    step = RunConfigLogObserver(
+        command=get_runconfig_command(posttrigger) + ['--stepname', stepname, '--phase', phase],
+        name=name,
+        description=description,
+        logfiles=get_buildlogs(maxsteps),
+        lazylogfiles=True,
+        maxsteps=maxsteps,
+        timeout=16200)  # default of 1200s/20min is too short, use 4.5hrs
+    return step
+
+class RunConfigCheckSteps(shell.ShellCommand):
+    name = 'Check run-config steps to use'
+    descriptionDone = ['Checked which run-config approach to use']
+    haltOnFailure = False
+    flunkOnFailure = True
+    jsonFileName = util.Interpolate("%(prop:builddir)s/runconfig.json")
+    logfiles = {'json': jsonFileName}
+
+    def __init__(self, *args, **kwargs):
+        self.posttrigger = kwargs.pop("posttrigger")
+        self.command = get_runconfig_command(self.posttrigger)
+        self.command.append("--json-outputfile")
+        self.command.append(self.jsonFileName)
+        super().__init__(*args, **kwargs)
+
+    def start(self):
+        self.log_observer_json = logobserver.BufferLogObserver()
+        self.addLogObserver('json', self.log_observer_json)
+        return super().start()
+
+    def evaluateCommand(self, cmd):
+        # If the command fails, fall back to old style run-config execution
+        rc = super().evaluateCommand(cmd)
+        jsonconfig = self.getProperty("runconfig-json", None)
+        if rc == FAILURE or not jsonconfig:
+            steps = [get_runconfig_legacy_step(self.posttrigger)]
+        else:
+            steps = []
+            for s in jsonconfig:
+                name = "run-config-" + s['name'] + "-" + s['phase']
+                steps.append(get_runconfig_step(name, s['name'], s['phase'], name, self.posttrigger))
+        self.build.addStepsAfterCurrentStep(steps)
+        return SUCCESS
+
+    def commandComplete(self, cmd):
+        super().commandComplete(cmd)
+        logLines = self.log_observer_json.getStdout()
+        json_text = ''.join([line for line in logLines.splitlines()])
+        if not json_text:
+            return
+        try:
+            jsonconfig = json.loads(json_text)
+            self.setProperty("runconfig-json", jsonconfig)
+        except Exception as ex:
+            self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
+
+    @defer.inlineCallbacks
+    def _addToLog(self, logName, message):
+        try:
+            log = self.getLog(logName)
+        except KeyError:
+            log = yield self.addLog(logName)
+        log.addStdout(message)