Add custom log observer

We want seperate log files in the UI for each of the build steps. This
changes buildbot to monitor the command.log.X files which run-config
generates. It also searches the log output for errors and warnings and
summarises these neatly in the UI.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

builders.py

@@ -2,6 +2,7 @@ from buildbot.plugins import *
 
 from yoctoabb import config
 from yoctoabb.steps.writelayerinfo import WriteLayerInfo
+from yoctoabb.steps.observer import RunConfigLogObserver
 from datetime import datetime
 
 import os
@@ -119,6 +120,11 @@ def ensure_props_set(props):
         "publish_destination": props.getProperty("publish_destination", "None")
     }
 
+def get_buildlogs():
+    logfiles = {}
+    for i in range(1,30):
+        logfiles["step" + str(i)] = "build/command.log." + str(i)
+    return logfiles
 
 def create_builder_factory():
     f = util.BuildFactory()
@@ -153,7 +159,8 @@ def create_builder_factory():
                                            haltOnFailure=True,
                                            name='Set build revision'))
 
-    f.addStep(steps.ShellCommand(
+
+    f.addStep(RunConfigLogObserver(
         command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
                  util.Property("buildername"),
                  util.Interpolate("%(prop:builddir)s/build/build"),
@@ -164,6 +171,8 @@ def create_builder_factory():
                  get_publish_dest,
                  util.URLForBuild],
         name="run-config",
+        logfiles=get_buildlogs(),
+        lazylogfiles=True,
         timeout=16200))  # default of 1200s/20min is too short, use 4.5hrs
     return f
 
@@ -225,7 +234,7 @@ factory.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop
                                              name='Set build revision'))
 
 # run-config
-factory.addStep(steps.ShellCommand(
+factory.addStep(RunConfigLogObserver(
     command=[
         util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
         util.Property("buildername"),
@@ -237,6 +246,8 @@ factory.addStep(steps.ShellCommand(
         get_publish_dest,
         util.URLForBuild],
     name="run-config",
+    logfiles=get_buildlogs(),
+    lazylogfiles=True,
     timeout=16200))  # default of 1200s/20min is too short, use 4.5hrs
 
 # trigger the buildsets contained in the nightly set

steps/observer.py

@@ -0,0 +1,49 @@
+from twisted.python import log
+
+from buildbot.process import logobserver
+from buildbot.process.results import FAILURE
+from buildbot.process.results import SKIPPED
+from buildbot.process.results import SUCCESS
+from buildbot.process.results import WARNINGS
+from buildbot.steps.shell import ShellCommand
+
+#
+# Monitor the step 1-X logs and stdio, collecting up any warnings and errors seen
+# and publish them at the end in their own 'logfile' for ease of access to the user
+#
+class RunConfigLogObserver(ShellCommand):
+
+    warnOnWarnings = True
+    warnOnFailure = True
+    warnings = 0
+    errors = 0
+
+    def __init__(self, python=None, *args, **kwargs):
+        ShellCommand.__init__(self, *args, **kwargs)
+        self.python = python
+        self.warningLines = []
+        self.errorLines = []
+        self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
+        for i in range(1, 30):
+            self.addLogObserver('step' + str(i), logobserver.LineConsumerLogObserver(self.logConsumer))
+
+    def logConsumer(self):
+        while True:
+            stream, line = yield
+            if line.startswith("WARNING:"):
+                self.warnings += 1
+                self.warningLines.append(line)
+            if line.startswith("ERROR:"):
+                self.errors += 1
+                self.errorLines.append(line)
+
+    def commandComplete(self, cmd):
+        self.addCompleteLog('warnings', '\n'.join(self.warningLines))
+        self.addCompleteLog('errors', '\n'.join(self.errorLines))
+
+    def evaluateCommand(self, cmd):
+        if cmd.didFail() or self.errors:
+            return FAILURE
+        if self.warnings:
+            return WARNINGS
+        return SUCCESS