mirror of
git://git.yoctoproject.org/yocto-autobuilder2.git
synced 2025-07-19 12:49:03 +02:00
builders: Move run-config handling to its own module
Also abstract the calls into a function to allow for easier further development. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie
parent
8cc03a2819
commit
97104f0a69
162
builders.py
162
builders.py
|
|
@ -2,10 +2,10 @@ from buildbot.plugins import *
|
|||
|
||||
from yoctoabb import config
|
||||
from yoctoabb.steps.writelayerinfo import WriteLayerInfo
|
||||
from yoctoabb.steps.runconfig import get_publish_dest, get_publish_resultdir, get_publish_name, get_runconfig_legacy_step
|
||||
from yoctoabb.steps.observer import RunConfigLogObserver
|
||||
|
||||
from twisted.python import log
|
||||
from datetime import datetime
|
||||
|
||||
import os
|
||||
import json
|
||||
|
|
@ -13,7 +13,6 @@ import random
|
|||
|
||||
|
||||
builders = []
|
||||
maxsteps = 9
|
||||
|
||||
# Environment to pass into the workers, e.g. to load further local configuration
|
||||
# fragments
|
||||
|
|
@ -21,95 +20,6 @@ extra_env = {}
|
|||
if os.environ.get('ABHELPER_JSON'):
|
||||
extra_env['ABHELPER_JSON'] = os.environ['ABHELPER_JSON']
|
||||
|
||||
@util.renderer
|
||||
def get_sstate_release_number(props):
|
||||
"""
|
||||
Uses the values submitted to the scheduler to determine the major number
|
||||
of the release for the purposes of publishing per-major release
|
||||
shared-state artefacts.
|
||||
"""
|
||||
release_number = props.getProperty("yocto_number")
|
||||
if not release_number:
|
||||
return ""
|
||||
release_components = release_number.split('.', 3)
|
||||
return '.'.join(release_components).strip('.')
|
||||
|
||||
|
||||
def get_publish_internal(props):
|
||||
"""
|
||||
Calculate the location to which artefacts should be published and store it
|
||||
as a property for use by other workers.
|
||||
"""
|
||||
# Cache the value in the publish_detination property
|
||||
dest = props.getProperty("publish_destination", "")
|
||||
if dest:
|
||||
return dest
|
||||
|
||||
if props.getProperty("is_release", False):
|
||||
milestone = props.getProperty("milestone_number", "")
|
||||
rc_number = props.getProperty("rc_number", "")
|
||||
snapshot = ""
|
||||
if milestone:
|
||||
snapshot += "_" + milestone
|
||||
if rc_number:
|
||||
snapshot += "." + rc_number
|
||||
|
||||
rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
|
||||
dest = os.path.join(config.publish_dest, "releases", rel_name)
|
||||
else:
|
||||
dest_base = os.path.join(config.publish_dest, 'non-release',
|
||||
datetime.now().strftime("%Y%m%d"))
|
||||
|
||||
# We want to make sure that we aren't writing artefacts to a publish
|
||||
# directory which already exists, therefore we keep a list of used
|
||||
# publish paths to prevent re-use. We store that in a JSON file.
|
||||
useddests = {}
|
||||
# NOTE: we make a strong assumption here that this code lives in a
|
||||
# directory which is an immediate child of the buildbot master's
|
||||
# working directory.
|
||||
basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"..")
|
||||
persist = os.path.join(basedir, "pub_locations.json")
|
||||
if os.path.exists(persist):
|
||||
with open(persist) as f:
|
||||
useddests = json.load(f)
|
||||
|
||||
rev = useddests.get(dest_base, "")
|
||||
if rev: # incremenent and use
|
||||
rev = int(rev) + 1
|
||||
else: # use the base path and store rev 0
|
||||
rev = 1
|
||||
dest = "%s-%s" % (dest_base, rev)
|
||||
# need to update the used destinations
|
||||
useddests[dest_base] = rev
|
||||
# save the info, overwriting the file if it exists
|
||||
with open(persist, 'w') as out:
|
||||
json.dump(useddests, out)
|
||||
|
||||
# set the destination as a property to be inherited by workers, so that
|
||||
# all workers in a triggered set publish to the same location
|
||||
props.setProperty("publish_destination", dest,
|
||||
"get_publish_dest")
|
||||
return dest
|
||||
|
||||
@util.renderer
|
||||
def get_publish_dest(props):
|
||||
deploy = props.getProperty("deploy_artefacts", False)
|
||||
if not deploy:
|
||||
return ""
|
||||
return get_publish_internal(props)
|
||||
|
||||
@util.renderer
|
||||
def get_publish_resultdir(props):
|
||||
return get_publish_internal(props) + "/testresults"
|
||||
|
||||
@util.renderer
|
||||
def get_publish_name(props):
|
||||
dest = get_publish_internal(props)
|
||||
if dest:
|
||||
return os.path.basename(dest)
|
||||
return dest
|
||||
|
||||
@util.renderer
|
||||
def ensure_props_set(props):
|
||||
"""
|
||||
|
|
@ -126,13 +36,6 @@ def ensure_props_set(props):
|
|||
"publish_destination": props.getProperty("publish_destination", "")
|
||||
}
|
||||
|
||||
def get_buildlogs(maxsteps):
|
||||
logfiles = {}
|
||||
for i in range(1, maxsteps):
|
||||
for j in ['a', 'b', 'c', 'd']:
|
||||
logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j)
|
||||
return logfiles
|
||||
|
||||
def create_builder_factory():
|
||||
f = util.BuildFactory()
|
||||
|
||||
|
|
@ -173,25 +76,9 @@ def create_builder_factory():
|
|||
haltOnFailure=True,
|
||||
name='Set build branch'))
|
||||
|
||||
f.addStep(RunConfigLogObserver(
|
||||
command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
|
||||
util.Property("buildername"),
|
||||
util.Interpolate("%(prop:builddir)s/build/build"),
|
||||
util.Interpolate("%(prop:branch_poky)s"),
|
||||
util.Interpolate("%(prop:repo_poky)s"),
|
||||
"--sstateprefix", get_sstate_release_number,
|
||||
"--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"),
|
||||
"--build-type", util.Interpolate("%(prop:build_type)s"),
|
||||
"--workername", util.Interpolate("%(prop:workername)s"),
|
||||
"--publish-dir", get_publish_dest,
|
||||
"--build-url", util.URLForBuild,
|
||||
"--results-dir", get_publish_resultdir,
|
||||
"--quietlogging"],
|
||||
name="run-config",
|
||||
logfiles=get_buildlogs(maxsteps),
|
||||
lazylogfiles=True,
|
||||
maxsteps=maxsteps,
|
||||
timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
|
||||
|
||||
f.addStep(get_runconfig_legacy_step(posttrigger=False))
|
||||
|
||||
return f
|
||||
|
||||
def nextWorker(bldr, workers, buildrequest):
|
||||
|
|
@ -294,25 +181,7 @@ def create_parent_builder_factory(buildername, waitname):
|
|||
name='Set build branch'))
|
||||
|
||||
# run-config
|
||||
factory.addStep(RunConfigLogObserver(
|
||||
command=[
|
||||
util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
|
||||
util.Property("buildername"),
|
||||
util.Interpolate("%(prop:builddir)s/build/build"),
|
||||
util.Interpolate("%(prop:branch_poky)s"),
|
||||
util.Interpolate("%(prop:repo_poky)s"),
|
||||
"--sstateprefix", get_sstate_release_number,
|
||||
"--publish-dir", get_publish_dest,
|
||||
"--build-type", util.Interpolate("%(prop:build_type)s"),
|
||||
"--workername", util.Interpolate("%(prop:workername)s"),
|
||||
"--build-url", util.URLForBuild,
|
||||
"--results-dir", get_publish_resultdir,
|
||||
"--quietlogging"],
|
||||
name="run-config",
|
||||
logfiles=get_buildlogs(maxsteps),
|
||||
lazylogfiles=True,
|
||||
maxsteps=maxsteps,
|
||||
timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
|
||||
factory.addStep(get_runconfig_legacy_step(posttrigger=False))
|
||||
|
||||
# trigger the buildsets contained in the nightly set
|
||||
def get_props_set():
|
||||
|
|
@ -342,26 +211,7 @@ def create_parent_builder_factory(buildername, waitname):
|
|||
waitForFinish=True,
|
||||
set_properties=get_props_set()))
|
||||
|
||||
factory.addStep(RunConfigLogObserver(
|
||||
command=[
|
||||
util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
|
||||
util.Interpolate("%(prop:buildername)s-posttrigger"),
|
||||
util.Interpolate("%(prop:builddir)s/build/build"),
|
||||
util.Interpolate("%(prop:branch_poky)s"),
|
||||
util.Interpolate("%(prop:repo_poky)s"),
|
||||
"--sstateprefix", get_sstate_release_number,
|
||||
"--publish-dir", get_publish_dest,
|
||||
"--build-type", util.Interpolate("%(prop:build_type)s"),
|
||||
"--workername", util.Interpolate("%(prop:workername)s"),
|
||||
"--build-url", util.URLForBuild,
|
||||
"--results-dir", get_publish_resultdir,
|
||||
"--quietlogging"],
|
||||
name="run-config",
|
||||
logfiles=get_buildlogs(maxsteps),
|
||||
lazylogfiles=True,
|
||||
maxsteps=maxsteps,
|
||||
timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
|
||||
|
||||
factory.addStep(get_runconfig_legacy_step(posttrigger=True))
|
||||
|
||||
factory.addStep(steps.ShellCommand(
|
||||
command=[
|
||||
|
|
|
|||
141
steps/runconfig.py
Normal file
141
steps/runconfig.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
from twisted.internet import defer
|
||||
from buildbot.plugins import steps, util
|
||||
from buildbot.process import buildstep
|
||||
from buildbot.process.results import Results, SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY
|
||||
|
||||
from yoctoabb.steps.observer import RunConfigLogObserver
|
||||
|
||||
import json
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
|
||||
from yoctoabb import config
|
||||
|
||||
maxsteps = 9
|
||||
|
||||
def get_publish_internal(props):
|
||||
"""
|
||||
Calculate the location to which artefacts should be published and store it
|
||||
as a property for use by other workers.
|
||||
"""
|
||||
# Cache the value in the publish_detination property
|
||||
dest = props.getProperty("publish_destination", "")
|
||||
if dest:
|
||||
return dest
|
||||
|
||||
if props.getProperty("is_release", False):
|
||||
milestone = props.getProperty("milestone_number", "")
|
||||
rc_number = props.getProperty("rc_number", "")
|
||||
snapshot = ""
|
||||
if milestone:
|
||||
snapshot += "_" + milestone
|
||||
if rc_number:
|
||||
snapshot += "." + rc_number
|
||||
|
||||
rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
|
||||
dest = os.path.join(config.publish_dest, "releases", rel_name)
|
||||
else:
|
||||
dest_base = os.path.join(config.publish_dest, 'non-release',
|
||||
datetime.now().strftime("%Y%m%d"))
|
||||
|
||||
# We want to make sure that we aren't writing artefacts to a publish
|
||||
# directory which already exists, therefore we keep a list of used
|
||||
# publish paths to prevent re-use. We store that in a JSON file.
|
||||
useddests = {}
|
||||
# NOTE: we make a strong assumption here that this code lives in a
|
||||
# directory which is an immediate child of the buildbot master's
|
||||
# working directory.
|
||||
basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"..")
|
||||
persist = os.path.join(basedir, "pub_locations.json")
|
||||
if os.path.exists(persist):
|
||||
with open(persist) as f:
|
||||
useddests = json.load(f)
|
||||
|
||||
rev = useddests.get(dest_base, "")
|
||||
if rev: # incremenent and use
|
||||
rev = int(rev) + 1
|
||||
else: # use the base path and store rev 0
|
||||
rev = 1
|
||||
dest = "%s-%s" % (dest_base, rev)
|
||||
# need to update the used destinations
|
||||
useddests[dest_base] = rev
|
||||
# save the info, overwriting the file if it exists
|
||||
with open(persist, 'w') as out:
|
||||
json.dump(useddests, out)
|
||||
|
||||
# set the destination as a property to be inherited by workers, so that
|
||||
# all workers in a triggered set publish to the same location
|
||||
props.setProperty("publish_destination", dest,
|
||||
"get_publish_dest")
|
||||
return dest
|
||||
|
||||
@util.renderer
|
||||
def get_publish_dest(props):
|
||||
deploy = props.getProperty("deploy_artefacts", False)
|
||||
if not deploy:
|
||||
return ""
|
||||
return get_publish_internal(props)
|
||||
|
||||
@util.renderer
|
||||
def get_publish_resultdir(props):
|
||||
return get_publish_internal(props) + "/testresults"
|
||||
|
||||
@util.renderer
|
||||
def get_publish_name(props):
|
||||
dest = get_publish_internal(props)
|
||||
if dest:
|
||||
return os.path.basename(dest)
|
||||
return dest
|
||||
|
||||
@util.renderer
|
||||
def get_sstate_release_number(props):
|
||||
"""
|
||||
Uses the values submitted to the scheduler to determine the major number
|
||||
of the release for the purposes of publishing per-major release
|
||||
shared-state artefacts.
|
||||
"""
|
||||
release_number = props.getProperty("yocto_number")
|
||||
if not release_number:
|
||||
return ""
|
||||
release_components = release_number.split('.', 3)
|
||||
return '.'.join(release_components).strip('.')
|
||||
|
||||
def get_runconfig_command(posttrigger=False):
|
||||
runconfig_command = [util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config")]
|
||||
if posttrigger:
|
||||
runconfig_command.append(util.Interpolate("%(prop:buildername)s-posttrigger"))
|
||||
else:
|
||||
runconfig_command.append(util.Property("buildername"))
|
||||
runconfig_command.extend([
|
||||
util.Interpolate("%(prop:builddir)s/build/build"),
|
||||
util.Interpolate("%(prop:branch_poky)s"),
|
||||
util.Interpolate("%(prop:repo_poky)s"),
|
||||
"--sstateprefix", get_sstate_release_number,
|
||||
"--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"),
|
||||
"--publish-dir", get_publish_dest,
|
||||
"--build-type", util.Interpolate("%(prop:build_type)s"),
|
||||
"--workername", util.Interpolate("%(prop:workername)s"),
|
||||
"--build-url", util.URLForBuild,
|
||||
"--results-dir", get_publish_resultdir,
|
||||
"--quietlogging"])
|
||||
return runconfig_command
|
||||
|
||||
def get_buildlogs(maxsteps):
|
||||
logfiles = {}
|
||||
for i in range(1, maxsteps):
|
||||
for j in ['a', 'b', 'c', 'd']:
|
||||
logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j)
|
||||
return logfiles
|
||||
|
||||
def get_runconfig_legacy_step(posttrigger):
|
||||
step = RunConfigLogObserver(
|
||||
command=get_runconfig_command(posttrigger),
|
||||
name="run-config",
|
||||
logfiles=get_buildlogs(maxsteps),
|
||||
lazylogfiles=True,
|
||||
maxsteps=maxsteps,
|
||||
timeout=16200) # default of 1200s/20min is too short, use 4.5hrs
|
||||
return step
|
||||
|
||||
Loading…
Reference in New Issue
Block a user