mirror of
git://git.yoctoproject.org/yocto-autobuilder2.git
synced 2025-07-19 03:49:11 +02:00
2856c3283d
The got_revision trick, whilst quite neat, doesn't quite work and is a bit of a hack breaking other buildbot internals. It gets et late in the builds leaving a confusing build display in the meantime. Use a dedicated property to store the "Yocto" build revision instead. This property will be used by our own custom console UI. Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
273 lines
11 KiB
Python
273 lines
11 KiB
Python
from buildbot.plugins import *
|
|
|
|
from yoctoabb import config
|
|
from yoctoabb.steps.writelayerinfo import WriteLayerInfo
|
|
from datetime import datetime
|
|
|
|
import os
|
|
import json
|
|
|
|
|
|
builders = []
|
|
|
|
|
|
@util.renderer
|
|
def get_sstate_release_number(props):
|
|
"""
|
|
Uses the values submitted to the scheduler to determine the major number
|
|
of the release for the purposes of publishing per-major release
|
|
shared-state artefacts.
|
|
"""
|
|
release_number = props.getProperty("yocto_number")
|
|
if not release_number:
|
|
return ""
|
|
release_components = release_number.split('.', 3)
|
|
return '.'.join(release_components).strip('.')
|
|
|
|
|
|
def get_publish_internal(props, basename=False):
|
|
"""
|
|
Calculate the location to which artefacts should be published and store it
|
|
as a property for use by other workers.
|
|
"""
|
|
dest = ""
|
|
deploy = props.getProperty("deploy_artifacts", "False")
|
|
if props.getProperty("deploy_artifacts", "False") != "False":
|
|
rel_name = ""
|
|
dest = props.getProperty("publish_destination", "")
|
|
if dest:
|
|
if basename:
|
|
return os.path.basename(dest)
|
|
return dest
|
|
|
|
if props.getProperty("is_release", "False") == "True":
|
|
milestone = props.getProperty("milestone_number", "")
|
|
rc_number = props.getProperty("rc_number", "")
|
|
snapshot = ""
|
|
if milestone:
|
|
snapshot += "_" + milestone
|
|
if rc_number:
|
|
snapshot += "." + rc_number
|
|
|
|
rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
|
|
dest = os.path.join(config.publish_dest, "releases", rel_name)
|
|
else:
|
|
dest_base = os.path.join(config.publish_dest,
|
|
props.getProperty("buildername"),
|
|
datetime.now().strftime("%Y%m%d"))
|
|
|
|
# We want to make sure that we aren't writing artefacts to a publish
|
|
# directory which already exists, therefore we keep a list of used
|
|
# publish paths to prevent re-use. We store that in a JSON file.
|
|
useddests = {}
|
|
# NOTE: we make a strong assumption here that this code lives in a
|
|
# directory which is an immediate child of the buildbot master's
|
|
# working directory.
|
|
basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
"..")
|
|
persist = os.path.join(basedir, "pub_locations.json")
|
|
if os.path.exists(persist):
|
|
with open(persist) as f:
|
|
useddests = json.load(f)
|
|
|
|
rev = useddests.get(dest_base, "")
|
|
if rev: # incremenent and use
|
|
rev = int(rev) + 1
|
|
else: # use the base path and store rev 0
|
|
rev = 1
|
|
dest = "%s-%s" % (dest_base, rev)
|
|
# need to update the used destinations
|
|
useddests[dest_base] = rev
|
|
# save the info, overwriting the file if it exists
|
|
with open(persist, 'w') as out:
|
|
json.dump(useddests, out)
|
|
|
|
# set the destination as a property to be inherited by workers, so that
|
|
# all workers in a triggered set publish to the same location
|
|
props.setProperty("publish_destination", dest,
|
|
"get_publish_dest")
|
|
if basename:
|
|
return os.path.basename(dest)
|
|
return dest
|
|
else:
|
|
return "None"
|
|
|
|
@util.renderer
|
|
def get_publish_dest(props):
|
|
return get_publish_internal(props, basename=False)
|
|
|
|
@util.renderer
|
|
def get_publish_name(props):
|
|
return get_publish_internal(props, basename=True)
|
|
|
|
@util.renderer
|
|
def ensure_props_set(props):
|
|
"""
|
|
When the various properties aren't set (i.e. a single builder was force
|
|
triggered instead of being triggered by the nightly) we need to ensure they
|
|
correct defaults are set and passed to the helper scripts.
|
|
"""
|
|
return {
|
|
"sharedrepolocation": props.getProperty("sharedrepolocation", "None"),
|
|
"is_release": props.getProperty("is_release", "None"),
|
|
"buildappsrcrev": props.getProperty("buildappsrcrev", "None"),
|
|
"deploy_artifacts": props.getProperty("deploy_artifacts", "False"),
|
|
"publish_destination": props.getProperty("publish_destination", "None")
|
|
}
|
|
|
|
|
|
def create_builder_factory():
|
|
f = util.BuildFactory()
|
|
|
|
# NOTE: Assumes that yocto-autobuilder repo has been cloned to home
|
|
# directory of the user running buildbot.
|
|
clob = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
|
|
f.addStep(steps.ShellCommand(
|
|
command=[clob, util.Interpolate("%(prop:builddir)s/")],
|
|
haltOnFailure=True,
|
|
name="Clobber build dir"))
|
|
f.addStep(steps.Git(
|
|
repourl='git://git.yoctoproject.org/yocto-autobuilder-helper',
|
|
workdir=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper"),
|
|
mode='incremental',
|
|
haltOnFailure=True,
|
|
name='Fetch yocto-autobuilder-helper'))
|
|
f.addStep(steps.SetProperties(properties=ensure_props_set))
|
|
f.addStep(WriteLayerInfo(name='Write main layerinfo.json', haltOnFailure=True))
|
|
f.addStep(steps.ShellCommand(
|
|
command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"),
|
|
util.Interpolate("%(prop:builddir)s/layerinfo.json"),
|
|
util.Interpolate("%(prop:sharedrepolocation)s"),
|
|
util.Interpolate("%(prop:builddir)s/build"),
|
|
util.Property("buildername"),
|
|
util.Property("is_release")],
|
|
haltOnFailure=True,
|
|
name="Unpack shared repositories"))
|
|
|
|
f.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop:sharedrepolocation)s/poky; git rev-parse HEAD"),
|
|
property="yp_build_revision",
|
|
haltOnFailure=True,
|
|
name='Set build revision'))
|
|
|
|
f.addStep(steps.ShellCommand(
|
|
command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
|
|
util.Property("buildername"),
|
|
util.Interpolate("%(prop:builddir)s/build/build"),
|
|
util.Interpolate("%(prop:branch_poky)s"),
|
|
util.Interpolate("%(prop:repo_poky)s"),
|
|
get_sstate_release_number,
|
|
util.Interpolate("%(prop:buildappsrcrev)s"),
|
|
get_publish_dest,
|
|
util.URLForBuild],
|
|
name="run-config",
|
|
timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
|
|
return f
|
|
|
|
|
|
# regular builders
|
|
f = create_builder_factory()
|
|
for builder in config.triggered_builders:
|
|
workers = config.builder_to_workers.get(builder, None)
|
|
if not workers:
|
|
workers = config.builder_to_workers['default']
|
|
builders.append(util.BuilderConfig(name=builder,
|
|
workernames=workers,
|
|
factory=f))
|
|
|
|
factory = util.BuildFactory()
|
|
# NOTE: Assumes that yocto-autobuilder repo has been cloned to home
|
|
# directory of the user running buildbot.
|
|
clob = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
|
|
factory.addStep(steps.ShellCommand(
|
|
command=[clob, util.Interpolate("%(prop:builddir)s/")],
|
|
haltOnFailure=True,
|
|
name="Clobber build dir"))
|
|
# check out the source
|
|
factory.addStep(steps.Git(
|
|
repourl='git://git.yoctoproject.org/yocto-autobuilder-helper',
|
|
workdir=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper"),
|
|
mode='incremental',
|
|
haltOnFailure=True,
|
|
name='Fetch yocto-autobuilder-helper'))
|
|
factory.addStep(WriteLayerInfo(name='Write main layerinfo.json', haltOnFailure=True))
|
|
factory.addStep(steps.ShellCommand(
|
|
command=[
|
|
util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/prepare-shared-repos"),
|
|
util.Interpolate("%(prop:builddir)s/layerinfo.json"),
|
|
util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
|
|
get_publish_dest],
|
|
haltOnFailure=True,
|
|
name="Prepare shared repositories"))
|
|
factory.addStep(steps.SetProperty(
|
|
property="sharedrepolocation",
|
|
value=util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir))
|
|
))
|
|
|
|
# shared-repo-unpack
|
|
factory.addStep(steps.ShellCommand(
|
|
command=[
|
|
util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"),
|
|
util.Interpolate("%(prop:builddir)s/layerinfo.json"),
|
|
util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
|
|
util.Interpolate("%(prop:builddir)s/build"),
|
|
util.Property("buildername"),
|
|
util.Property("is_release")],
|
|
haltOnFailure=True,
|
|
name="Unpack shared repositories"))
|
|
|
|
factory.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop:sharedrepolocation)s/poky; git rev-parse HEAD"),
|
|
property="yp_build_revision",
|
|
haltOnFailure=True,
|
|
name='Set build revision'))
|
|
|
|
# run-config
|
|
factory.addStep(steps.ShellCommand(
|
|
command=[
|
|
util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
|
|
util.Property("buildername"),
|
|
util.Interpolate("%(prop:builddir)s/build/build"),
|
|
util.Interpolate("%(prop:branch_poky)s"),
|
|
util.Interpolate("%(prop:repo_poky)s"),
|
|
get_sstate_release_number,
|
|
"None",
|
|
get_publish_dest,
|
|
util.URLForBuild],
|
|
name="run-config",
|
|
timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
|
|
|
|
# trigger the buildsets contained in the nightly set
|
|
def get_props_set():
|
|
set_props = {
|
|
"sharedrepolocation": util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
|
|
"is_release": util.Property("is_release"),
|
|
"buildappsrcrev": "None",
|
|
"deploy_artifacts": util.Property("deploy_artifacts"),
|
|
"publish_destination": util.Property("publish_destination"),
|
|
}
|
|
|
|
for repo in config.repos:
|
|
set_props["branch_%s" % repo] = util.Property("branch_%s" % repo)
|
|
set_props["commit_%s" % repo] = util.Property("commit_%s" % repo)
|
|
set_props["repo_%s" % repo] = util.Property("repo_%s" % repo)
|
|
|
|
return set_props
|
|
|
|
factory.addStep(steps.Trigger(schedulerNames=['wait'],
|
|
waitForFinish=True,
|
|
set_properties=get_props_set()))
|
|
|
|
factory.addStep(steps.ShellCommand(
|
|
command=[
|
|
util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/send-qa-email"),
|
|
util.Interpolate("%(prop:builddir)s/layerinfo.json"),
|
|
get_publish_dest,
|
|
get_publish_name,
|
|
util.Interpolate("%(prop:sharedrepolocation)s")
|
|
],
|
|
name="Send QA Email"))
|
|
|
|
builders.append(
|
|
util.BuilderConfig(name="nightly",
|
|
workernames=config.workers,
|
|
factory=factory))
|