diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..e7d6f99 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,114 @@ +FROM ubuntu:24.04 + +ARG AUTOBUILDER2_BRANCH=master + +USER root + +# Install needed packages +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && \ + apt full-upgrade -y && \ + apt -y install \ + build-essential \ + chrpath \ + cpio \ + debianutils \ + diffstat \ + file \ + gawk \ + gcc \ + git \ + iproute2 \ + iputils-ping \ + libacl1 \ + liblz4-tool \ + locales \ + npm \ + python3 \ + python3-git \ + python3-jinja2 \ + python3-pexpect \ + python3-pip \ + python3-subunit \ + socat \ + texinfo \ + unzip \ + virtualenv \ + wget \ + xz-utils \ + zstd + +RUN pip3 install --break-system-packages \ + buildbot==4.2.1 \ + buildbot-www==4.2.1 \ + buildbot-waterfall-view==4.2.1 \ + buildbot-console-view==4.2.1 \ + buildbot-grid-view==4.2.1 \ + buildbot-worker==4.2.1 \ + buildbot_pkg==4.2.1 \ + websockets + +RUN npm install -g yarn + +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +RUN ssh-keyscan push.yoctoproject.org push.openembedded.org >> /etc/ssh/ssh_known_hosts + +# Setup buildbot +COPY yocto-autobuilder2_patches /yocto-autobuilder2_patches +RUN useradd -m --system pokybuild && \ + cd /home/pokybuild && \ + buildbot create-master -r yocto-controller && \ + buildbot-worker create-worker -r --umask=0o22 yocto-worker controller example-worker pass && \ + \ + cd yocto-controller && \ + git clone https://git.yoctoproject.org/git/yocto-autobuilder2 yoctoabb && \ + git -C yoctoabb apply /yocto-autobuilder2_patches/*.patch && \ + ln -rs yoctoabb/master.cfg master.cfg && \ + \ + cd /home/pokybuild && \ + git clone https://git.yoctoproject.org/git/yocto-autobuilder-helper && \ + pip install --break-system-packages \ + yocto-controller/yoctoabb/yocto_console_view/ && \ + mkdir -p /home/pokybuild/git/ && \ + mkdir -p /home/pokybuild/tmp/ && \ + echo "[user]\nname=pokybuild\nemail=pokybuild@none\n" > \ + /home/pokybuild/.gitconfig && \ + chown -R pokybuild:nogroup /home/pokybuild && \ + mkdir -p /srv/autobuilder/ && \ + chown -R pokybuild:nogroup /srv/autobuilder/ + +# Fix config files for local runner +RUN sed -i \ + # Add example-worker \ + -e "s/^\(workers_debian *=.*\).*/\1 + [\"example-worker\"]/" \ + # Remove workers_bringup from all_workers \ + -e "/^all_workers *=/s/workers_bringup + //" \ + # Switch from ssh:// URLS to git:// so we do not need any auth \ + -e "s#ssh://git@push\.\(yoctoproject.org\|openembedded.org\)/#git://git.\1/#" \ + # Change location of sharedrepodir and publish_dest directories \ + -e "s@^\(sharedrepodir *= *\).*@\1\"/sharedrepo\"@" \ + -e "s@^\(publish_dest *= *\).*@\1\"/publish\"@" \ + /home/pokybuild/yocto-controller/yoctoabb/config.py && \ + sed -i \ + # Change location of sharedrepodir and publish_dest directories \ + -e "s#ssh://git@push\.\(yoctoproject.org\|openembedded.org\)/#git://git.\1/#" \ + /home/pokybuild/yocto-autobuilder-helper/config.json && \ + sed -i \ + # imp was removed in python 3.12, switch to importlib \ + -e "s/^import imp$/import importlib as imp/" \ + # Fix own URL \ + -e "s@^\(c\['buildbotURL'\] *= *\).*@\1\"http://localhost:8010/\"@" \ + /home/pokybuild/yocto-controller/master.cfg && \ + # Comment any sched.Nightly schedulers \ + # We try to support multilines code blocks by counting parenthesis \ + awk '/^schedulers.append\(sched.Nightly\(/{comment=1} /\(/{count+=gsub("\\(", "(")} /\)/{count-=gsub("\\)", ")")} {if (comment) {print "#" $0} else {print} if (count==0) comment=0}' \ + /home/pokybuild/yocto-controller/yoctoabb/schedulers.py + +COPY entry.sh / +RUN chmod a+x /entry.sh + +ENTRYPOINT ["/entry.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..c75f482 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,23 @@ +# Local autobuilder setup docker + +This docker compose configuration aims to provide a working minimal setup, +allowing to test and debug the buildbot configuration. + +It create two docker containers based on the same image, one acting as a +buildbot controller, one acting as buildbot worker. + +The buildbot configuration will be modified in several ways from the +configuration used in the public autobuilders. Main changes include: + - Only one worker will be used. + - Git urls will be modified to use git protocol instead of ssh, removing needs + for authentication. + - All nightly schedulers will be disabled. + +## Usage + +The local autobuilder can be started by running `docker-compose up` in this +folder. Once the dockers are started, buildbot web interface will be exposed on +http://localhost:8010/. + +You might want to modify the `compose.yaml` file first to suit your needs, such +as the cpu count and memory limits. diff --git a/docker/compose.yaml b/docker/compose.yaml new file mode 100644 index 0000000..71758e9 --- /dev/null +++ b/docker/compose.yaml @@ -0,0 +1,36 @@ +# Expose buildbot on http://localhost:8010/ + +--- + +services: + controller: + build: . + volumes: + - sharedrepo:/sharedrepo + - publish:/publish + - mirror:/home/pokybuild/git/mirror + - srvautobuilder:/srv/autobuilder + ports: + - 8010:8010 + stdin_open: true + tty: true + cpus: 4 + command: controller + worker: + build: . + volumes: + - sharedrepo:/sharedrepo + - publish:/publish + - mirror:/home/pokybuild/git/mirror + - srvautobuilder:/srv/autobuilder + stdin_open: true + tty: true + cpus: 8 + command: worker yocto-worker + mem_limit: 16gb + +volumes: + sharedrepo: + publish: + mirror: + srvautobuilder: diff --git a/docker/entry.sh b/docker/entry.sh new file mode 100644 index 0000000..3518eec --- /dev/null +++ b/docker/entry.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +chown pokybuild:nogroup /sharedrepo +chown pokybuild:nogroup /publish +chown pokybuild:nogroup /home/pokybuild/git/mirror +chown pokybuild:nogroup /srv/autobuilder +cd /home/pokybuild/ || exit 1 + +role="$1" + +if [ "${role}" = "controller" ]; then + su pokybuild -c "yocto-autobuilder-helper/janitor/ab-janitor" & + su pokybuild -c "buildbot start yocto-controller" + #tail -F yocto-controller/twistd.log & +elif [ "${role}" = "worker" ]; then + workername="$2" + su pokybuild -c "buildbot-worker start ${workername}" +else + echo "Unexpected role: ${role}" + exit 2 +fi + +/bin/bash diff --git a/docker/yocto-autobuilder2_patches/0001-builders-Do-not-create-tags-during-builds.patch b/docker/yocto-autobuilder2_patches/0001-builders-Do-not-create-tags-during-builds.patch new file mode 100644 index 0000000..567b80f --- /dev/null +++ b/docker/yocto-autobuilder2_patches/0001-builders-Do-not-create-tags-during-builds.patch @@ -0,0 +1,25 @@ +From ee671836ad9f0b65fa8b8b11c7f366377521ab20 Mon Sep 17 00:00:00 2001 +From: Mathieu Dubois-Briand +Date: Fri, 3 Jan 2025 09:39:06 +0100 +Subject: [PATCH 1/2] builders: Do not create tags during builds + +Signed-off-by: Mathieu Dubois-Briand +--- + builders.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/builders.py b/builders.py +index 14ea1df1192e..ba47241de2cc 100644 +--- a/builders.py ++++ b/builders.py +@@ -273,7 +273,6 @@ def create_parent_builder_factory(buildername, waitname): + util.Interpolate("%(prop:builddir)s/layerinfo.json"), + util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)), + "-p", get_publish_dest, +- "-t", util.Interpolate("%(prop:buildername)s-%(prop:buildnumber)s"), + ], + haltOnFailure=True, + name="Prepare shared repositories")) +-- +2.39.5 + diff --git a/docker/yocto-autobuilder2_patches/0002-builders-Modify-free-space-check-to-work-in-docker.patch b/docker/yocto-autobuilder2_patches/0002-builders-Modify-free-space-check-to-work-in-docker.patch new file mode 100644 index 0000000..6d3ec55 --- /dev/null +++ b/docker/yocto-autobuilder2_patches/0002-builders-Modify-free-space-check-to-work-in-docker.patch @@ -0,0 +1,36 @@ +From 9bf9680ab9125138d926c22ca434ad86053bb5c4 Mon Sep 17 00:00:00 2001 +From: Mathieu Dubois-Briand +Date: Wed, 20 Nov 2024 14:53:34 +0100 +Subject: [PATCH 2/2] builders: Modify free space check to work in docker + +Signed-off-by: Mathieu Dubois-Briand +--- + builders.py | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/builders.py b/builders.py +index ba47241de2cc..02f6c10c6add 100644 +--- a/builders.py ++++ b/builders.py +@@ -67,8 +67,7 @@ def canStartBuild(builder, wfb, request): + + # threshold is GB of space + checks = { +- "." : (200, "HOME"), +- "/tmp" : (1, "/tmp"), ++ "." : (200, "root"), + } + + for mountpoint in checks: +@@ -76,7 +75,7 @@ def canStartBuild(builder, wfb, request): + + threshold = threshold * 1024 *1024 * 1024 + +- cmd = yield shell("findmnt -T %s --df -n --bytes | awk '{print $5}'" % mountpoint, wfb.worker, builder) ++ cmd = yield shell("findmnt --all -T %s --df -n --bytes | awk '{print $5}'" % mountpoint, wfb.worker, builder) + if int(cmd.stdout) < threshold: + log.msg("Detected {0} GB of space available on {1}, less than threshold of {2} GB. Can't start build".format(cmd.stdout, name, threshold)) + wfb.worker.quarantine_timeout = 10 * 60 +-- +2.39.5 +