From 4fd189ab38f4f169a7195dbbc7dd4c87d207b573 Mon Sep 17 00:00:00 2001 From: Joshua Lock Date: Thu, 22 Feb 2018 10:38:19 +0000 Subject: [PATCH] Initial prototype of using yocto-autobuilder-helper scripts Initial prototype of using yocto-autobuilder-helper scripts from vanilla buildbot to replicate yocto-autobuilder configuration. * README.md is updated to describe goals and approach * TODO contains known issues and work items, TODO: comments in the code point to specific locations of work Signed-off-by: Joshua Lock --- COPYING | 340 +++++++++++++++++++++++++++++++++ README.md | 89 +++++++++ TODO | 15 ++ builders.py | 167 +++++++++++++++++ config.py | 81 ++++++++ lib/__init__.py | 0 lib/wiki.py | 206 ++++++++++++++++++++ master.cfg | 45 +++++ reporters/__init__.py | 0 reporters/wikilog.py | 407 ++++++++++++++++++++++++++++++++++++++++ schedulers.py | 145 ++++++++++++++ services.py | 33 ++++ steps/__init__.py | 0 steps/writelayerinfo.py | 48 +++++ workers.py | 9 + www.py | 23 +++ 16 files changed, 1608 insertions(+) create mode 100644 COPYING create mode 100644 README.md create mode 100644 TODO create mode 100644 builders.py create mode 100644 config.py create mode 100644 lib/__init__.py create mode 100644 lib/wiki.py create mode 100644 master.cfg create mode 100644 reporters/__init__.py create mode 100644 reporters/wikilog.py create mode 100644 schedulers.py create mode 100644 services.py create mode 100644 steps/__init__.py create mode 100644 steps/writelayerinfo.py create mode 100644 workers.py create mode 100644 www.py diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e98507a --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# yoctoabb +Prototype of using yocto-autobuilder-helper from vanilla buildbot to replicate yocto-autobuilder configuration + +## Introduction +The goal of this repository is to provide a buildbot configuration for use with +the yocto-autobuilder-helper[1] scripts which has as little code and as few +custom buildbot extensions as possible. The configuration merely collects +enough inputs from the user to furnish the yocto-autobuilder-helper scripts +with sufficient inputs to do their work. + +The configuration was written for the latest (at time of writing) buildbot 1.0 +release. + +## Overview +The role of this buildbot configuration is simple, we want to provide +sufficient user-customisable parameters to trigger the yocto-autobuilder-helpers +build scripts. + +Each builder, mapping to a named configuration in yocto-autobuilder-helper, is +created with steps and properties required to invoke the helper scripts in the +expected fashion. + +We create custom schedulers for each builder with parameters configured on the +schedulers which can supply custom versions of the required values for the +yocto-autobuilder-helper script parameters. + +### Code layout +builders.py -- configures the builders with minimal buildsteps to invoke the yocto-autobuilder-helper scripts +lib/ + wiki.py -- implements some mediawiki related functionality as used by the wikilog plugin +reporters/ + wikilog.py -- our custom plugin to write info on build failures to a wiki page +steps/ + writelayerinfo.py -- write the user supplied (or default) repos to a JSON file for use by the scripts +config.py -- goal is to contain all values that might need changing to redeploy this code elsewhere. Goal hasn't yet been met. +master.cfg -- calls into other scripts to do most configuration. Cluster specific config still lives here (i.e. controller url). +schedulers.py -- sets up the force schedulers with controls for modifying inputs for each builder. +services.py -- configures irc, mail and wikilog` reporters. +workers.py -- configures the worker objects +www.py -- sets up the web UI + +## Customisations +Whilst the goal is as little custom code as possible, there were some +customisations required both in order to support the yocto-autobuilder-helper +workflows and to replicate the workflows established with the outgoing +yocto-autobuilder[2]. + +### WriteLayerInfo buildstep +steps/writelayerinfo.py -- implements a simple custom buildset to iterate the +repo_, branch_, and commit_ properties set by the schedulers and write a JSON +file with the user's values. + +### WikiLog reporter +reporters/wikilog.py -- a buildbot service to listen for build failures and +write some information on them to the configured wiki page. +lib/wiki.py -- some helper functions for the wiki plugin, much of this code can +be replaced by porting the plugin to be a buildbot.util.service.HTTPClient +implementation + +## Deployment +### Upstream Yocto Project autobuilder +__on the controller__ +``` +$ buildbot create-master +$ cd +$ git clone +$ cd .. +$ ln -rs controller/yoctoab/master.cfg controller/master.cfg +$ $EDITOR controller/yoctoab/master.cfg + +$ $EDITOR controller/yoctoab/services.py + +$ $EDITOR controller/yoctoab/www.py + +``` + +__on the worker__ +``` +$ buildbot-worker create-worker +``` + +NOTE: the 3rd parameter to create-worker, the worker name, need not be +hard-coded, for example pass `hostname` to use the host's configured name + +### None upstream users +__TODO__: requires a custom config.json for yocto-autobuilder-helper + +1. http://git.yoctoproject.org/clean/cgit.cgi/yocto-autobuilder-helper +2. http://git.yoctoproject.org/clean/cgit.cgi/yocto-autobuilder diff --git a/TODO b/TODO new file mode 100644 index 0000000..858479d --- /dev/null +++ b/TODO @@ -0,0 +1,15 @@ +# Initial release +* TODO items, esp in builders.py i.e finishing up nightly builder config +* formatting/labels of the yocto-autobuilder-helper inputs + +# Future +* add mail notification functionality to yocto-autobuilder-helper, it already + knows how to iterate error reports. +* Move clobber script, janitor, etc into yocto-autobuilder-helper and make use of them +* Simple script to start buildbot controller, janitor and PRServer +* Review ForceScheduler inputs and whether they are used by the helper scripts, particularly the poky_number parameter of nightly is possibly no longer required +* Teach the janitor to pre-clone the repos +* add SendQAMail +* Look into allowed_origins property of built in web server +* add buildhistory trigger +* switch wikilog to buildbot.util.service.HTTPClient? diff --git a/builders.py b/builders.py new file mode 100644 index 0000000..f158f64 --- /dev/null +++ b/builders.py @@ -0,0 +1,167 @@ +from buildbot.plugins import * + +from yoctoab import config +from yoctoab.steps.writelayerinfo import WriteLayerInfo + +import os + + +builders = [] + + +def get_sstate_release_number(): + # TODO: implement + # release_number = util.Interpolate("%(prop:yocto_number)s") + # if not release_number: + # return "" + # release_components = release_number.split('.', 3) + # return '.'.join(release_components).strip('.') + return "None" + + +def get_publish_dest(): + # if deploy_artifacts property is False return None + return "None" # FIXME: based on SetDest? + + +@util.renderer +def ensure_props_set(props): + """ + When the various properties aren't set (i.e. a single builder was force + triggered instead of being triggered by the nightly) we need to ensure they + correct defaults are set and passed to the helper scripts. + """ + return { + "sharedrepolocation": props.getProperty("sharedrepolocation", "None"), + "is_release": props.getProperty("is_release", "None"), + "buildappsrcrev": props.getProperty("buildappsrcrev", "None") + } + + +def create_builder_factory(): + f = util.BuildFactory() + + # FIXME: use RP's script + f.addStep(steps.ShellCommand( + command=["rm", "-fr", util.Interpolate("%(prop:builddir)s/")], + name="Clobber build dir")) + f.addStep(steps.Git( + repourl='git://git.yoctoproject.org/yocto-autobuilder-helper', + workdir=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper"), + mode='incremental', + name='Fetch yocto-autobuilder-helper')) + f.addStep(steps.SetProperties(properties=ensure_props_set)) + f.addStep(WriteLayerInfo(name='Write main layerinfo.json')) + f.addStep(steps.ShellCommand( + command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"), + util.Interpolate("%(prop:builddir)s/layerinfo.json"), + util.Interpolate("%(prop:sharedrepolocation)s"), + util.Interpolate("%(prop:builddir)s/build"), + util.Property("buildername"), + util.Property("is_release")], + name="Unpack shared repositories")) + f.addStep(steps.ShellCommand( + command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"), + util.Property("buildername"), + util.Interpolate("%(prop:builddir)s/build/build"), + util.Interpolate("%(prop:branch_poky)s"), + util.Interpolate("%(prop:repo_poky)s"), + get_sstate_release_number(), + util.Interpolate("%(prop:buildappsrcrev)s"), + get_publish_dest(), + util.URLForBuild], + name="run-config")) + return f + + +# regular builders +f = create_builder_factory() +for builder in config.triggered_builders: + workers = config.builder_to_workers.get(builder, None) + if not workers: + workers = config.builder_to_workers['default'] + builders.append(util.BuilderConfig(name=builder, + workernames=workers, + factory=f)) + +factory = util.BuildFactory() +# FIXME: use RP's script +factory.addStep(steps.ShellCommand( + command=["rm", "-fr", util.Interpolate("%(prop:builddir)s/")], + name="Clobber build dir")) +# check out the source +factory.addStep(steps.Git( + repourl='git://git.yoctoproject.org/yocto-autobuilder-helper', + workdir=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper"), + mode='incremental', + name='Fetch yocto-autobuilder-helper')) +factory.addStep(WriteLayerInfo(name='Write main layerinfo.json')) +factory.addStep(steps.ShellCommand( + command=[ + util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/prepare-shared-repos"), + util.Interpolate("%(prop:builddir)s/layerinfo.json"), + util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)), + config.publish_dest], + name="Prepare shared repositories")) +factory.addStep(steps.SetProperty( + property="sharedrepolocation", + value=util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)) +)) + +# shared-repo-unpack +factory.addStep(steps.ShellCommand( + command=[ + util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"), + util.Interpolate("%(prop:builddir)s/layerinfo.json"), + util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)), + util.Interpolate("%(prop:builddir)s/build"), + util.Property("buildername"), + util.Property("is_release")], + name="Unpack shared repositories")) + +# run-config +factory.addStep(steps.ShellCommand( + command=[ + util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"), + util.Property("buildername"), + util.Interpolate("%(prop:builddir)s/build/build"), + util.Interpolate("%(prop:branch_poky)s"), + util.Interpolate("%(prop:repo_poky)s"), + get_sstate_release_number(), + "None", + get_publish_dest(), + util.URLForBuild], + name="run-config")) + +# trigger the buildsets contained in the nightly set +set_props = { + "sharedrepolocation": util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)), + "is_release": util.Property("is_release"), + "buildappsrcrev": "None", + "branch_poky": util.Property("branch_poky"), + "commit_poky": util.Property("commit_poky"), + "repo_poky": util.Property("repo_poky") +} +factory.addStep(steps.Trigger(schedulerNames=['nowait'], + waitForFinish=False, + set_properties=set_props)) +factory.addStep(steps.Trigger(schedulerNames=['wait'], + waitForFinish=True, + set_properties=set_props)) + +# selftest +factory.addStep(steps.ShellCommand( + command=". ./oe-init-build-env; bitbake-selftest", + workdir=util.Interpolate("%(prop:builddir)s/build") +)) + +# TODO: trigger buildhistory_nowait - possibly no longer required? + +# TODO: send QA mail if a release - compose and pass to sendmail command? + +# TODO: update Current Link if published + +builders.append( + util.BuilderConfig(name="nightly", + workernames=config.workers, + factory=factory)) diff --git a/config.py b/config.py new file mode 100644 index 0000000..a2c2285 --- /dev/null +++ b/config.py @@ -0,0 +1,81 @@ +# ## Build configuration, tied to config.json in yocto-autobuilder-helpers +# Repositories used by each builder +buildertorepos = { + "eclipse": ["eclipse-poky-neon", "eclipse-poky-oxygen"], + "nightly": ["poky", "meta-intel", "openembedded-core", "bitbake", + "eclipse-poky-neon", "eclipse-poky-oxygen", "meta-qt4", + "meta-qt3", "meta-mingw", "meta-gplv2"], + "nightly-qa-extras": ["poky", "meta-mingw"], + "nightly-oecore": ["openembedded-core", "bitbake"], + "nightly-checkuri": ["poky", "meta-qt4", "meta-qt3"], + "default": ["poky"] +} + +# Repositories used that the scripts need to know about and should be buildbot +# user customisable +repos = { + "yocto-autobuilder-helper": + ["git://git.yoctoproject.org/yocto-autobuilder-helper", + "master"], + "eclipse-poky-neon": ["git://git.yoctoproject.org/eclipse-poky", + "neon-master"], + "eclipse-poky-oxygen": ["git://git.yoctoproject.org/eclipse-poky", + "oxygen-master"], + "poky": ["git://git.yoctoproject.org/poky", "master"], + "meta-intel": ["git://git.yoctoproject.org/meta-intel", "master"], + "openembedded-core": ["git://git.openembedded.org/openembedded-core", + "master"], + "bitbake": ["git://git.openembedded.org/bitbake", "master"], + "meta-qt4": ["git://git.yoctoproject.org/meta-qt4", "master"], + "meta-qt3": ["git://git.yoctoproject.org/meta-qt3", "master"], + "meta-mingw": ["git://git.yoctoproject.org/meta-mingw", "master"], + "meta-gplv2": ["git://git.yoctoproject.org/meta-gplv2", "master"] +} + +trigger_builders_wait = [ + "nightly-arm", "nightly-arm-lsb", "nightly-arm64", + "nightly-mips", "nightly-mips-lsb", "nightly-mips64", + "nightly-multilib", "nightly-x32", + "nightly-ppc", "nightly-ppc-lsb" + "nightly-x86-64", "nightly-x86-64-lsb", + "nightly-x86", "nightly-x86-lsb", + "nightly-packagemanagers", + "nightly-rpm-non-rpm", "nightly-deb-non-deb" +] +trigger_builders_nowait = [ + "build-appliance", "buildtools", "eclipse-plugin-neon", + "eclipse-plugin-oxygen", "nightly-non-gpl3", "nightly-oecore", + "nightly-world", "nightly-world-wic", "nightly-world-lsb", + "poky-tiny", "nightly-musl", "nightly-musl-x86-64", "nightly-no-x11", + "nightly-qa-extras", "nightly-oe-selftest" +] + +triggered_builders = trigger_builders_wait + trigger_builders_nowait +builders = ["nightly"] + triggered_builders + +# Supported Yocto Project releases, by name +releases = ["", "sumo", "rocko", "pyro", "morty"] + +# ## Cluster configuration +# Publishing settings +sharedrepodir = "/srv/www/vhosts/repos.yoctoproject.org" +publish_dest = "/srv/www/vhosts/autobuilder.yoctoproject.org/pub" + +# Web UI settings +web_port = 8010 + +# List of workers in the cluster +workers = ["example-worker"] + +# Worker configuration, all workers configured the same... +# TODO: support per-worker config +worker_password = "pass" +worker_max_builds = None +notify_on_missing = None + +# Some builders should only run on specific workers (host OS dependent) +builder_to_workers = { + "nightly-rpm-non-rpm": [], + "nightly-deb-non-deb": [], + "default": workers +} diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/wiki.py b/lib/wiki.py new file mode 100644 index 0000000..7bdd4e3 --- /dev/null +++ b/lib/wiki.py @@ -0,0 +1,206 @@ +''' +Created on Dec 13, 2016 + +__author__ = "Joshua Lock" +__copyright__ = "Copyright 2016, Intel Corp." +__credits__ = ["Joshua Lock"] +__license__ = "GPL" +__version__ = "2.0" +__maintainer__ = "Joshua Lock" +__email__ = "joshua.g.lock@intel.com" +''' + +import codecs +import hashlib +import time +import requests +from twisted.python import log + + +class YPWiki(object): + MAX_TRIES = 5 + TIMEOUT = 60 + + def __init__(self, wiki_uri, wiki_un, wiki_pass): + self.wiki_uri = wiki_uri + self.wiki_un = wiki_un + self.wiki_pass = wiki_pass + + @staticmethod + def retry_request(requesturl, **kwargs): + """ + Rather than failing when a request to a 'requesturl' throws an + exception retry again a minute later. Perform this retry no more than + 5 times. + + @type requesturl: string + """ + kwargs['timeout'] = YPWiki.TIMEOUT + + def try_request(): + try: + req = requests.get(requesturl, **kwargs) + return req + except (requests.exceptions.RequestException, + requests.exceptions.Timeout): + return None + + tries = 0 + req = None + while not req and tries < YPWiki.MAX_TRIES: + if tries > 0: + time.sleep(60) + req = try_request() + tries = tries + 1 + + return req + + @staticmethod + def parse_json(response): + """ + This method handles stripping UTF-8 BOM from the beginning of responses + from the Yocto Project wiki. + + http://en.wikipedia.org/wiki/Byte_Order_Mark + http://bugs.python.org/issue18958 + + @type response: requests.Response + """ + bom = codecs.BOM_UTF8 + text = '' + + # In Requests 0.8.2 (Ubuntu 12.04) Response.content has type unicode, + # whereas in requests 2.1.10 (Fedora 23) Response.content is a str + # Ensure that bom is the same type as the content, codecs.BOM_UTF8 is + # a str + if type(response.content) == unicode: + bom = unicode(codecs.BOM_UTF8, 'utf8') + + # If we discover a BOM set the encoding appropriately so that the + # built in decoding routines in requests work correctly. + if response.content.startswith(bom): + response.encoding = 'utf-8-sig' + + return response.json() + + def login(self): + """ + Login to the wiki and return cookies for the logged in session + """ + payload = { + 'action': 'login', + 'lgname': self.wiki_un, + 'lgpassword': self.wiki_pass, + 'utf8': '', + 'format': 'json' + } + + try: + req1 = requests.post(self.wiki_uri, data=payload, + timeout=self.TIMEOUT) + except (requests.exceptions.RequestException, + requests.exceptions.Timeout): + return None + + parsed = self.parse_json(req1) + login_token = parsed['login']['token'].encode('utf-8') + + payload['lgtoken'] = login_token + try: + req2 = requests.post(self.wiki_uri, data=payload, + cookies=req1.cookies, timeout=self.TIMEOUT) + except (requests.exceptions.RequestException, + requests.exceptions.Timeout): + return None + + return req2.cookies.copy() + + def get_content(self, wiki_page): + """ + Get the current content of the 'wiki_page' -- to make the wiki page + as useful as possible the most recent log entry should be at the top, + to that end we need to edit the whole page so that we can insert the + new entry after the log but before the other entries. + + This method fetches the current page content, splits out the blurb and + returns a pair: + 1) the blurb + 2) the current entries + + @type wiki_page: string + """ + + pm = '?format=json&action=query&prop=revisions&rvprop=content&titles=' + + req = self.retry_request(self.wiki_uri+pm+wiki_page) + if not req: + return None, None + + parsed = self.parse_json(req) + pageid = sorted(parsed['query']['pages'].keys())[-1].encode('utf-8') + content = parsed['query']['pages'][pageid]['revisions'][0]['*'] + content = content.encode('utf-8') + blurb, entries = content.split('==', 1) + # ensure we keep only a single newline after the blurb + blurb = blurb.strip() + "\n" + entries = '=='+entries + + return blurb, entries + + def post_entry(self, wiki_page, content, summary, cookies): + """ + Post the new page contents 'content' to the page title 'wiki_page' + with a 'summary' using the login credentials from 'cookies' + + @type wiki_page: string + @type content: string + @type summary: string + @type cookies: CookieJar + """ + + params = ("?format=json&action=query&prop=info|revisions" + "&intoken=edit&rvprop=timestamp&titles=") + req = self.retry_request(self.wiki_uri+params+wiki_page, + cookies=cookies) + if not req: + return False + + parsed = self.parse_json(req) + pageid = sorted(parsed['query']['pages'].keys())[-1].encode('utf-8') + edit_token = parsed['query']['pages'][pageid]['edittoken'] + edit_token = edit_token.encode('utf-8') + + edit_cookie = cookies.copy() + edit_cookie.update(req.cookies) + + content_hash = hashlib.md5(content).hexdigest() + + payload = { + 'action': 'edit', + 'assert': 'user', + 'title': wiki_page, + 'summary': summary, + 'text': content, + 'md5': content_hash, + 'token': edit_token, + 'utf8': '', + 'format': 'json' + } + + try: + req = requests.post(self.wiki_uri, data=payload, + cookies=edit_cookie, timeout=self.TIMEOUT) + except (requests.exceptions.RequestException, + requests.exceptions.Timeout): + return False + + if not req.status_code == requests.codes.ok: + log.err("Unexpected status code %s received when trying to post" + " an entry to the wiki." % req.status_code) + return False + else: + result = self.parse_json(req) + status = result.get('edit', {}).get('result', '').encode('utf-8') + if status == 'Success': + return True + return False diff --git a/master.cfg b/master.cfg new file mode 100644 index 0000000..3dd278c --- /dev/null +++ b/master.cfg @@ -0,0 +1,45 @@ +# -*- python -*- +# ex: set filetype=python: + +from buildbot.plugins import * +from yoctoab import builders, schedulers, workers, services, www + +import imp + + +# supports re-loading configuration with buildbot sighup, meaning we can change +# configuration without having to restart buildbot. +# Note: code modules (in lib/, steps/ and reporters/) are not reloaded with a +# buildbot sighup +imp.reload(config) +imp.reload(builders) +imp.reload(schedulers) +imp.reload(workers) +imp.reload(services) +imp.reload(www) + +c = BuildmasterConfig = {} + +# Disable usage reporting +c['buildbotNetUsageData'] = None +c['protocols'] = {'pb': {'port': 9989}} +c['db'] = {'db_url' : "sqlite:///state.sqlite",} + +# Yocto Autobuilder doesn't have a change source, all builds +# are manually triggered +c['change_source'] = [] + +# Items which are common to Yocto Project autobuilder deployments using the +# yocto-autobuilder-helper scripts +c['schedulers'] = schedulers.schedulers +c['builders'] = builders.builders +c['services'] = services.services +c['www'] = www.www + +# These items are specific to an individual AB deployment +c['workers'] = workers.workers + +c['title'] = "Yocto Autobuilder" +c['titleURL'] = "https://autobuilder.yoctoproject.org/main/" +# visible location for internal web server +c['buildbotURL'] = "https://autobuilder.yoctoproject.org/main/" diff --git a/reporters/__init__.py b/reporters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/reporters/wikilog.py b/reporters/wikilog.py new file mode 100644 index 0000000..cca99d7 --- /dev/null +++ b/reporters/wikilog.py @@ -0,0 +1,407 @@ +from buildbot.reporters import utils +from buildbot.util import service +from twisted.internet import defer +from twisted.python import log + +from yoctoab.lib.wiki import YPWiki + + +class WikiLog(service.BuildbotService): + name = "WikiLog" + wiki = None + + def checkConfig(self, wiki_uri, wiki_un, wiki_pass, wiki_page, + identifier=None, **kwargs): + service.BuildbotService.checkConfig(self) + + @defer.inlineCallbacks + def reconfigService(self, wiki_uri, wiki_un, wiki_pass, wiki_page, + identifier=None, **kwargs): + yield service.BuildbotService.reconfigService(self) + self.wiki_page = wiki_page + tagfmt = " on {}" + if identifier: + self.tag = tagfmt(identifier) + else: + self.tag = "" + self.wiki = YPWiki(wiki_uri, wiki_un, wiki_pass) + + @defer.inlineCallbacks + def startService(self): + yield service.BuildbotService.startService(self) + + startConsuming = self.master.mq.startConsuming + self._buildCompleteConsumer = yield startConsuming( + self.buildFinished, + ('builds', None, 'finished')) + + self._buildStartedConsumer = yield startConsuming( + self.buildStarted, + ('builds', None, 'new')) + + # TODO: stepFinished? Or do we no longer need that now that the build + # is much simpler? + + def stopService(self): + self._buildCompleteConsumer.stopConsuming() + self._buildStartedConsumer.stopConsuming() + + def buildStarted(self, key, build): + yield utils.getDetailsForBuild(self.master, build, + **self.needed) + builderName = build['builder']['name'] + # TODO: this all seems a bit overly complex? + if builderName != "nightly" and not self.isTriggered(build)\ + and not self.isNightlyRunning(): + # If a build has been started which isn't nightly and a nightly + # build isn't running, chances are a single builder has been + # started in order to test something and it just finished. + if not self.logBuild(build): + log.err("wkl: Failed to log build %s on %s" % ( + build.getNumber(), builderName)) + elif builderName == "nightly": + if not self.logBuild(build): + log.err("wkl: Failed to log build %s on %s" % ( + build.getNumber(), builderName)) + + def stepFinished(self, key, build): + blurb, content = self.wiki.get_content(self.wiki_page) + if not blurb: + log.err("wkl: Couldn't get wiki content in stepFinished()") + return + update = self.updateBuildInfo(content, build) + + if not update: + # log.msg("wkl: No update, nothing to POST") + return + + if content == update: + # log.msg("wkl: No update made, no need to make a POST") + return + + cookies = self.wiki.login() + if not cookies: + log.err("wkl: Failed to login to wiki") + return + + summary = "Updating branch and commitish for %s" %\ + build.getNumber() + if not self.wiki.post_entry(self.wiki_page, blurb+update, summary, + cookies): + # log.err("wkl: Failed to update wikilog with summary: '{}'".format + # (summary)) + return + + def buildFinished(self, key, build): + # TODO: we don't have a result var + if result == SUCCESS: + return + + if not self.updateBuild(build): + log.err("wkl: Failed to update wikilog with build %s failure" % + build.getNumber()) + + def logBuild(self, build): + """ + Extract information about 'build' and post an entry to the wiki + + @type build: buildbot.status.build.BuildStatus + """ + + builder = build.getBuilder().getName() + reason = build.getReason() + buildid = str(build.getNumber()) + start, _ = build.getTimes() + url = self.status.getURLForThing(build) + buildbranch = build.getProperty('branch').strip() + if not buildbranch or len(buildbranch) < 1: + buildbranch = "YP_BUILDBRANCH" + chash = build.getProperty('commit_poky').strip() + if not chash or len(chash) < 1 or chash == "HEAD": + chash = "YP_CHASH" + + reason_list = reason.split(':', 1) + forcedby = reason_list[0].strip() + description = 'No reason given.' + if len(reason_list) > 1 and reason_list[1] != ' ': + description = reason_list[1].strip() + starttime = time.ctime(start) + + sectionfmt = '==[{} {} {} - {} {}]==' + section_title = sectionfmt.format(url, builder, buildid, buildbranch, + chash) + summaryfmt = 'Adding new BuildLog entry for build %s (%s)' + summary = summaryfmt % (buildid, chash) + summary = summary + self.tag + content = "* '''Build ID''' - %s" % chash + content = content + self.tag + "\n" + content = content + '* Started at: %s\n' % starttime + content = content + '* ' + forcedby + '\n* ' + description + '\n' + new_entry = '{}\n{}\n'.format(section_title, content).encode('utf-8') + + blurb, entries = self.wiki.get_content(self.wiki_page) + if not blurb: + log.err("wkl: Unexpected content retrieved from wiki!") + return False + + entries = new_entry + entries + cookies = self.wiki.login() + + if not cookies: + log.err("wkl: Failed to login to wiki") + return False + + if not self.wiki.post_entry(self.wiki_page, blurb+entries, + summary, cookies): + log.err("wkl: Failed to post entry for %s" % buildid) + return False + + log.msg("wkl: Posting wikilog entry for %s" % buildid) + return True + + def updateEntryBuildInfo(self, entry, build): + """ + Extract the branch and commit hash from the properties of the 'build' + and update the 'entry' string with extracted values + + @type entry: string + @type build: buildbot.status.build.BuildStatus + """ + # We only want to update the commit and branch info for the + # primary poky build + # FIXME: this is quite poky specific. Can we handle this in + # a more generic manner? + repo = build.getProperty("repourl_poky") + if not repo: + return entry + buildbranch = build.getProperty('branch').strip() + if not buildbranch or len(buildbranch) < 1: + buildbranch = "YP_BUILDBRANCH" + chash = build.getProperty('commit_poky').strip() + if not chash or len(chash) < 1 or chash == "HEAD": + chash = "YP_CHASH" + + new_entry = entry.replace("YP_BUILDBRANCH", buildbranch, 1) + new_entry = new_entry.replace("YP_CHASH", chash, 2) + + return new_entry + + def updateBuildInfo(self, content, build): + """ + Extract the branch and commit hash from the properties of the 'build' + and update the 'content' string with extracted values + + @type content: string + @type build: buildbot.status.build.BuildStatus + """ + + # Try to find an entry that matches this build, rather than blindly + # updating all instances of the template value in the content + buildid = build.getProperty('buildnumber', '0') + builder = build.getProperty('buildername', 'nobuilder') + entry_list = re.split('\=\=\[(.+)\]\=\=', content) + title_idx = -1 + # Start at the beginning of entry list and keep iterating until we find + # a title which looks ~right + for idx, ent in enumerate(entry_list): + # The matched title contents should always start with a http* + # schemed URI + if ent.startswith('http'): + # format of the title is: + # ==[url builder buildid - buildbranch commit_hash]== + title_components = ent.split(None, 6) + if builder == title_components[1] and \ + str(buildid) == title_components[2]: + title_idx = idx + break + + if title_idx < 0: + errmsg = ("wkl: Failed to update entry for {0} couldn't find a " + "matching title with builder {1}") + log.err(errmsg.format(buildid, builder)) + return content + + entry = entry_list[title_idx + 1] + title = entry_list[title_idx] + + combined = "==[{0}]=={1}".format(title, entry) + new_entry = self.updateEntryBuildInfo(combined, build) + new_entry = new_entry.encode('utf-8') + + it = re.finditer('\=\=\[(.+)\]\=\=', content) + entry_title = it.next() + while entry_title.group(1) != title: + entry_title = it.next() + next_title = it.next() + head = content[:entry_title.start()] + tail = content[next_title.start():] + update = head + new_entry + tail + + # log.msg("wkl: Updating commit info YP_BUILDBRANCH=%s YP_CHASH=%s" % + # (buildbranch, chash)) + + return update + + def updateBuild(self, build): + """ + Extract information about 'build' and update an entry in the wiki + + @type build: buildbot.status.build.BuildStatus + """ + builder = build.getBuilder().getName() + buildid = str(build.getNumber()) + reason = build.getReason() + blurb, entries = self.wiki.get_content(self.wiki_page) + if not blurb: + log.err("wkl: Unexpected content retrieved from wiki!") + return False + + url = self.status.getURLForThing(build) + log_entries = [] + logfmt = '[%s %s]' + for l in build.getLogs(): + # Ignore logs for steps which succeeded + result, _ = l.getStep().getResults() + if result == SUCCESS: + continue + + step_name = l.getStep().getName() + log_url = '%s/steps/%s/logs/%s' % (url, + step_name, + l.getName()) + log_url = log_url.replace(' ', '%20') + log_entry = logfmt % (log_url, step_name) + log_entries.append(log_entry) + buildbranch = build.getProperty('branch').strip() + if not buildbranch or len(buildbranch) < 1: + buildbranch = "YP_BUILDBRANCH" + chash = build.getProperty('commit_poky').strip() + if not chash or len(chash) < 1 or chash == "HEAD": + chash = "YP_CHASH" + + entry_list = re.split('\=\=\[(.+)\]\=\=', entries) + entry = '' + title = '' + # Start at the beginning of entry list and keep iterating until we find + # a title which looks ~right + trigger = "Triggerable(trigger_main-build" + for idx, entry in enumerate(entry_list): + # The matched title contents should always start with a http* + # schemed URI + if entry.startswith('http'): + # format of the title is: + # ==[url builder buildid - buildbranch commit_hash]== + title_components = entry.split(None, 6) + + # For the primary, nightly, builder we can match on chash and + # buildbranch, otherwise we have to hope that the first + # triggered build with matching chash and tag + foundmatch = False + if buildbranch == title_components[4] \ + and chash == title_components[5] \ + and self.tag in entry_list[idx+1]: + foundmatch = True + elif trigger in reason \ + and chash == title_components[5] \ + and self.tag in entry_list[idx+1]: + foundmatch = True + + if foundmatch: + entry = entry_list[idx+1] + title = entry_list[idx] + break + + if not entry or not title: + errmsg = ("wkl: Failed to update entry for {0} couldn't find a " + "matching title for branch: {1} or hash: {2} " + "(reason was '{3}')") + log.err(errmsg.format(buildid, buildbranch, chash, reason)) + return False + + log_fmt = '' + logs = '' + new_entry = '' + if builder == 'nightly': + # for failures in nightly we just append extra entries to the + # bullet list pointing to the failure logs + if len(log_entries) > 0: + logs = '\n* '.join(log_entries) + '\n' + new_entry = '\n' + entry.strip() + '\n* ' + logs + else: + # We only update the buildlog for a nightly build if there + # are additional items to append to the log list. + return True + else: + # for non-nightly failures we create an entry in the list linking + # to the failed builder and indent the logs as a child bullet list + log_fmt = '\n* ' + builderfmt = log_fmt + if self.isTriggered(build) or self.isNightlyRunning(): + log_fmt = '\n** ' + builderfmt = '\n* [%s %s] failed' % (url, builder) + + if len(log_entries) > 0: + if self.isTriggered(build) or self.isNightlyRunning(): + builderfmt = builderfmt + ': ' + log_fmt + logs = log_fmt.join(log_entries) + logs = logs + '\n' + new_entry = '\n' + entry.strip() + builderfmt + logs + + summary = 'Updating entry with failures in %s' % builder + summary = summary + self.tag + + new_entry = self.updateEntryBuildInfo(new_entry, build) + new_entry = new_entry.encode('utf-8') + + # Find the point where the first entry's title starts and the second + # entry's title begins, then replace the text between those points + # with the newly generated entry. + it = re.finditer('\=\=\[(.+)\]\=\=', entries) + entry_title = it.next() + while entry_title.group(1) != title: + entry_title = it.next() + next_title = it.next() + head = entries[:entry_title.end()] + tail = entries[next_title.start():] + update = head + new_entry + tail + + cookies = self.wiki.login() + if not cookies: + log.err("wkl: Failed to login to wiki") + return False + + if not self.wiki.post_entry(self.wiki_page, blurb+update, summary, + cookies): + log.err("wkl: Failed to update entry for %s" % buildid) + return False + + log.msg("wkl: Updating wikilog entry for %s" % buildid) + return True + + def isNightlyRunning(self): + """ + Determine whether there's a nightly build in progress + """ + nightly = self.master.getBuilder("nightly") + if not nightly: + return False + build = nightly.getBuild(-1) # the most recent build + if not build: + return False + running = not build.isFinished() + return running + + def isTriggered(self, build): + """ + build.isFinished() can return True when buildsteps triggered by + nightly are still running, therefore we provide a method to check + whether the 'build' was triggered by a nightly build. + + @type build: buildbot.status.build.BuildStatus + """ + reason = build.getReason() + reason_list = reason.split(':', 1) + forcedby = reason_list[0].strip() + if forcedby.startswith("Triggerable"): + return True + return False diff --git a/schedulers.py b/schedulers.py new file mode 100644 index 0000000..f86fc50 --- /dev/null +++ b/schedulers.py @@ -0,0 +1,145 @@ +from buildbot.plugins import schedulers as sched +from buildbot.plugins import util +from yoctoab import config + +schedulers = [] + + +def create_repo_inputs(reponame): + """ + Given the name of a repository in yoctoab.config's repo dict creates + StringParameter inputs to allow specification of alternative uri, branch + and commit/hash values + """ + + # TODO: formatting? Typically had branch & commit on same line? + repo = util.StringParameter("repo_{}".format(reponame), + label="{} repository".format(reponame), + default=config.repos[reponame][0]) + branch = util.StringParameter("branch_{}".format(reponame), + label="Branch", + default=config.repos[reponame][1]) + commit = util.StringParameter("commit_{}".format(reponame), + label="Tag/Commit hash", + default="HEAD") + return [repo, branch, commit] + + +def repos_for_builder(buildername): + """ + Returns a list of additional properties for a scheduler, a list of + StringParameter allowing all of the repositories used by the + builder/scheduler to be customised + """ + + parameters = [] + repos = config.buildertorepos.get(buildername) + if not repos: + repos = config.buildertorepos["default"] + for repo in repos: + inputs = create_repo_inputs(repo) + parameters = parameters + inputs + return parameters + + +def props_for_builder(builder): + """ + Generate an appropriate list of properties to use on a builder-specific + scheduler + """ + + props = [] + if builder == 'build-appliance': + props.append(util.StringParameter( + name="buildappsrcrev", + # TODO: is this statement still true? + label="""Build appliance src revision. Use DEFAULT to take the + srcrev currently in the recipe:""", + default="None", + )) + if builder in ['build-appliance', 'buildtools']: + props.append(util.ChoiceStringParameter( + name="deploy_artifacts", + label="Do we want to deploy artifacts? ", + choices=["False", "True"], + default="False" + )) + + props = props + repos_for_builder(builder) + return props + + +for builder in config.triggered_builders: + schedulers.append(sched.ForceScheduler( + name=builder, + builderNames=[builder], + reason=util.StringParameter( + name="reason", + label="""Reason (please note the reason for triggering the + build and any expectations for the build's outcome:""", + required=False), + properties=props_for_builder(builder), + buttonName="Force Build")) + +# nightly builder triggers various other builders +wait = sched.Triggerable(name="wait", + builderNames=config.trigger_builders_wait) +schedulers.append(wait) +nowait = sched.Triggerable(name="nowait", + builderNames=config.trigger_builders_nowait) +schedulers.append(nowait) + +schedulers.append(sched.ForceScheduler( + name="nightly", + builderNames=["nightly"], + buttonName="Start Nightly Build", + reason=util.StringParameter( + name="reason", + label="""Reason (please note the reason for triggering the build and + any expectations for the build's outcome:""", + required=False), + properties=[ + util.ChoiceStringParameter( + name="is_release", + label="Generate a release?", + choices=["False", "True"], + default="False"), + util.ChoiceStringParameter( + name="poky_name", + label="Name of the Poky release.", + choices=config.releases, + default=""), + util.ChoiceStringParameter( + name="is_milestone", + label="Is the release a milestone release?", + choices=["False", "True"], + default="False"), + util.StringParameter( + name="poky_number", + label="Poky Release Number (10.0.0, 11.0.1 etc.)"), + util.StringParameter( + name="yocto_number", + label="Yocto Project Release Number (1.5, 1.6 etc.)"), + util.ChoiceStringParameter( + name="milestone_number", + label="Milestone number", + choices=["", "M1", "M2", "M3", "M4"], + default="" + ), + util.ChoiceStringParameter( + name="rc_number", + label="Release candidate number", + choices=["", "rc1", "rc2", "rc3", "rc4", "rc5", "rc6", "rc7", + "rc8", "rc9"], + default=""), + util.ChoiceStringParameter( + name="send_email", + label="Send QA alert emails?", + choices=["False", "True"], + default="False"), + util.ChoiceStringParameter( + name="deploy_artifacts", + label="Do we want to deploy artifacts? ", + choices=["False", "True"], + default="False") + ]+repos_for_builder("nightly"))) diff --git a/services.py b/services.py new file mode 100644 index 0000000..f8e46dd --- /dev/null +++ b/services.py @@ -0,0 +1,33 @@ +from buildbot.plugins import reporters + +from yoctoab import config + + +services = [] + +# TODO: we'll replace this with functionality in yocto-autobuilder-helpers +# to mail the error reports to the list +# services.append( +# reporters.MailNotifier(fromaddr="yocto-builds@yoctoproject.org", +# sendToInterestedUsers=False, +# extraRecipients=["yocto-builds@yoctoproject.org"], +# mode=('failing',)) +# ) + +# services.append( +# reporters.IRC(host="irc.freenode.net", +# nick="YoctoAutobuilderBot", +# password="" +# notify_events={ +# 'successToFailure': 1, +# 'failureToSuccess': 0 +# }, +# channels=["yocto"], +# noticeOnChannel=True)) + +# from yoctoab.reporters import wikilog +# services.append( +# wikilog.WikiLog("https://wiki.yoctoproject.org/wiki/api.php", +# "User", "password", "LogPage" +# "production cluster") +# ) diff --git a/steps/__init__.py b/steps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/steps/writelayerinfo.py b/steps/writelayerinfo.py new file mode 100644 index 0000000..61a0bed --- /dev/null +++ b/steps/writelayerinfo.py @@ -0,0 +1,48 @@ +from twisted.internet import defer +from buildbot.process import buildstep +import json +import os + +from yoctoab import config + + +class WriteLayerInfo(buildstep.ShellMixin, buildstep.BuildStep): + name = "WriteLayerInfo" + + def __init__(self, **kwargs): + buildstep.BuildStep.__init__(self, **kwargs) + + def generateLayerInfo(self): + layerinfo = {} + writerepos = config.buildertorepos.get(self.getProperty("buildername")) + if not writerepos: + writerepos = config.buildertorepos["default"] + + for repo in writerepos: + repodict = {} + repodict["url"] = self.getProperty("repo_{}".format(repo)) + repodict["branch"] = self.getProperty("branch_{}".format(repo)) + repodict["revision"] = self.getProperty("commit_{}".format(repo)) + layerinfo[repo] = repodict + + return json.dumps(layerinfo, sort_keys=True, indent=4, + separators=(',', ': ')) + + @defer.inlineCallbacks + def run(self): + repojson = self.generateLayerInfo() + layerinfo = os.path.join(self.getProperty("builddir"), + "layerinfo.json") + writerepos = "printf '%s' >> %s" % (repojson, layerinfo) + cmd = yield self.makeRemoteShellCommand( + command=writerepos) + yield self.runCommand(cmd) + defer.returnValue(cmd.results()) + + +@defer.inlineCallbacks +def run(self): + cmd = RemoteCommand(args) + log = yield self.addLog('output') + cmd.useLog(log, closeWhenFinished=True) + yield self.runCommand(cmd) diff --git a/workers.py b/workers.py new file mode 100644 index 0000000..1643812 --- /dev/null +++ b/workers.py @@ -0,0 +1,9 @@ +from buildbot.plugins import worker +from yoctoab import config + +workers = [] + +for w in config.workers: + workers.append(worker.Worker(w, config.worker_password, + max_builds=config.worker_max_builds, + notify_on_missing=config.notify_on_missing)) diff --git a/www.py b/www.py new file mode 100644 index 0000000..32926d0 --- /dev/null +++ b/www.py @@ -0,0 +1,23 @@ +from buildbot.plugins import util +from yoctoab import config + + +# allow = [] +# +# for builder in config.builders: +# allow.append(util.ForceBuildEndpointMatcher(builder=builder, role='*')) +# allow.append(util.StopBuildEndpointMatcher(builder=builder, role='*')) +# +# authz = util.Authz( +# stringsMatcher=util.fnmatchStrMatcher, +# allowRules=allow, +# roleMatchers=[]) +# auth = util.HTPasswdAuth('/home/pokybuild/.htpasswd') +# +# www = dict(port=config.web_port, +# auth=auth, +# authz=authz, +# plugins=dict(waterfall_view={}, console_view={}, grid_view={})) + +www = dict(port=config.web_port, + plugins=dict(waterfall_view={}, console_view={}, grid_view={}))