mirror of
git://git.yoctoproject.org/yocto-autobuilder2.git
synced 2026-01-27 11:01:23 +01:00
679ef3780e
The new log observer added a few weeks ago fails to correctly show links to failed ptest logs. It looks like some unexpected newline character is messing with the parsing. Signed-off-by: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
125 lines
4.5 KiB
Python
125 lines
4.5 KiB
Python
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import re
|
|
|
|
from twisted.internet import defer
|
|
from twisted.python import log
|
|
|
|
from buildbot.process import logobserver
|
|
from buildbot.process.results import FAILURE
|
|
from buildbot.process.results import SKIPPED
|
|
from buildbot.process.results import SUCCESS
|
|
from buildbot.process.results import WARNINGS
|
|
from buildbot.steps.shell import ShellCommand
|
|
|
|
from functools import partial
|
|
|
|
#
|
|
# Monitor the step 1-X logs and stdio, collecting up any warnings and errors seen
|
|
# and publish them at the end in their own 'logfile' for ease of access to the user
|
|
#
|
|
class SimpleLogObserver(ShellCommand):
|
|
|
|
warnOnWarnings = True
|
|
warnOnFailure = True
|
|
warnCount = 0
|
|
errorCount = 0
|
|
|
|
def __init__(self, maxsteps=10, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.builder = None
|
|
self.warningLines = []
|
|
self.errorLines = []
|
|
self.links = []
|
|
self.failed_ptests = []
|
|
self.testResultsLink = None
|
|
if "description" in kwargs:
|
|
self.description = kwargs["description"]
|
|
else:
|
|
self.description = "run-config"
|
|
self.addLogObserver('stdio', logobserver.LineConsumerLogObserver(partial(self.logConsumer, 'stdio')))
|
|
self.yocto_io_re = re.compile(".*/([^/]*yocto.io/pub/(non-release|repro-fail[^/]*/)([^ ']|$)+).*")
|
|
self.ptest_fails_re = re.compile(r"WARNING: (\S*)-[0-9.]*-r[0-9]* do_testimage:")
|
|
|
|
def describe(self, done=False):
|
|
return self.description
|
|
|
|
def logConsumer(self, logname):
|
|
while True:
|
|
stream, line = yield
|
|
if line.startswith("WARNING:"):
|
|
self.warnCount += 1
|
|
self.warningLines.append(logname + ": " + line)
|
|
match = self.ptest_fails_re.match(line)
|
|
if match:
|
|
self.failed_ptests.append(match.group(1))
|
|
if line.startswith("ERROR:"):
|
|
self.errorCount += 1
|
|
self.errorLines.append(logname + ": " + line)
|
|
if line.startswith("Builder:"):
|
|
self.builder = line.split(':')[1].strip()
|
|
url, matched = self.yocto_io_re.subn('https://\\1', line.strip())
|
|
if matched:
|
|
self.links.append(url)
|
|
if not self.testResultsLink and url.endswith("/testresults"):
|
|
self.testResultsLink = url
|
|
|
|
@defer.inlineCallbacks
|
|
def finish_logs(self):
|
|
stdio_log = yield self.getLog('stdio')
|
|
yield stdio_log.finish()
|
|
|
|
@defer.inlineCallbacks
|
|
def createSummary(self):
|
|
if self.warnCount:
|
|
yield self.addCompleteLog('warnings', '\n'.join(self.warningLines) + '\n')
|
|
if self.errorLines:
|
|
yield self.addCompleteLog('errors', '\n'.join(self.errorLines) + '\n')
|
|
if self.links:
|
|
# Remove duplicates but preserve order
|
|
links = list(dict.fromkeys(self.links))
|
|
htmlLinks = ['Found links:<ul>']
|
|
htmlLinks.extend([f'<li><a href="{link}">{link}</a></li>' for link in links])
|
|
|
|
if self.testResultsLink and self.builder:
|
|
htmlLinks.extend(['</ul>ptest logs:<ul>'])
|
|
ptestLinks = [f"{self.testResultsLink}/{self.builder}/{ptest}"
|
|
for ptest in self.failed_ptests]
|
|
htmlLinks.extend([f'<li><a href="{link}">{link}</a></li>' for link in ptestLinks])
|
|
htmlLinks.append('</ul>')
|
|
yield self.addHTMLLog('links', '\n'.join(htmlLinks))
|
|
|
|
warnings_stat = self.getStatistic('warnings', 0)
|
|
self.setStatistic('warnings', warnings_stat + self.warnCount)
|
|
|
|
old_count = self.getProperty("warnings-count", 0)
|
|
self.setProperty(
|
|
"warnings-count", old_count + self.warnCount, "SimpleLogObserver")
|
|
|
|
@defer.inlineCallbacks
|
|
def run(self):
|
|
cmd = yield self.makeRemoteShellCommand()
|
|
yield self.runCommand(cmd)
|
|
|
|
yield self.finish_logs()
|
|
yield self.createSummary()
|
|
return self.evaluateCommand(cmd)
|
|
|
|
def evaluateCommand(self, cmd):
|
|
if cmd.didFail() or self.errorCount:
|
|
return FAILURE
|
|
if self.warnCount:
|
|
return WARNINGS
|
|
return SUCCESS
|
|
|
|
class RunConfigLogObserver(SimpleLogObserver):
|
|
|
|
def __init__(self, maxsteps=10, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
for i in range(1, maxsteps):
|
|
for j in ['a', 'b', 'c', 'd']:
|
|
name = 'step' + str(i) + str(j)
|
|
self.addLogObserver(name, logobserver.LineConsumerLogObserver(partial(self.logConsumer, name)))
|