docker: Allow to add several local workers

Add more more workers to test more realistic setups.

The first one is still added as a Debian builder, additional ones are
used as generic ones, allowing to test worker selection.

Signed-off-by: Mathieu Dubois-Briand <mathieu.dubois-briand@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

docker/Dockerfile

@@ -9,6 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update && \
     apt full-upgrade -y && \
     apt -y install \
+        bind9-host \
         build-essential \
         chrpath \
         cpio \
@@ -63,7 +64,6 @@ COPY . /yocto-autobuilder2
 RUN useradd -m --system pokybuild && \
     cd /home/pokybuild && \
     buildbot create-master -r yocto-controller && \
-    buildbot-worker create-worker -r --umask=0o22 yocto-worker controller example-worker pass && \
     \
     cd yocto-controller && \
     mv /yocto-autobuilder2 yoctoabb && \
@@ -85,8 +85,9 @@ RUN useradd -m --system pokybuild && \
 
 # Fix config files for local runner
 RUN sed -i \
-        # Add example-worker \
-        -e "s/^\(workers_debian *=.*\).*/\1 + [\"example-worker\"]/" \
+        # Add local workers \
+        -e "s/^\(workers_debian *=.*\).*/\1 + [f\"local-worker-debian\"]/" \
+        -e "s/^\(workers *=.*\).*/\1 + [f\"local-worker-extra-{i}\" for i in range(1, 9)]/" \
         # Remove workers_bringup from all_workers \
         -e "/^all_workers *=/s/workers_bringup + //" \
         # Switch from ssh:// URLS to git:// so we do not need any auth \

docker/README.md

@@ -8,7 +8,7 @@ buildbot controller, one acting as buildbot worker.
 
 The buildbot configuration will be modified in several ways from the
 configuration used in the public autobuilders. Main changes include:
- - Only one worker will be used.
+ - Only a few workers will be used.
  - Git urls will be modified to use git protocol instead of ssh, removing needs
    for authentication.
  - All nightly schedulers will be disabled.
@@ -17,7 +17,9 @@ configuration used in the public autobuilders. Main changes include:
 
 The local autobuilder can be started by running `docker-compose up` in this
 folder. Once the dockers are started, buildbot web interface will be exposed on
-http://localhost:8010/.
+http://localhost:8010/. Note that with the default profile, only one worker will
+be started. You can start extra workers with `docker-compose --profile
+manyworkers`.
 
 You might want to modify the `compose.yaml` file first to suit your needs, such
-as the cpu count and memory limits.
+as the cpu count, memory limits and number of extra workers.

docker/compose.yaml

@@ -20,12 +20,18 @@ services:
     ports:
       - 8010:8010
     cpus: 4
-    command: controller
-  worker:
+  worker: &base-worker
     <<: *base-service
     cpus: 8
-    command: worker yocto-worker
     mem_limit: 16gb
+  extraworker:
+    <<: *base-worker
+    cpus: 1
+    mem_limit: 2gb
+    deploy:
+      replicas: 8
+    profiles:
+      - manyworkers
 
 volumes:
   sharedrepo:

docker/entry.sh

@@ -6,15 +6,23 @@ chown pokybuild:nogroup /home/pokybuild/git/mirror
 chown pokybuild:nogroup /srv/autobuilder
 cd /home/pokybuild/ || exit 1
 
-role="$1"
+docker_name=$(host "$(host "$(hostname)" | awk '{print $NF}')" | awk '{print $NF}' | awk -F . '{print $1}')
+role=$(echo "${docker_name}" | cut -d_ -f 2)
+instance=$(echo "${docker_name}" | cut -d_ -f 3)
 
 if [ "${role}" = "controller" ]; then
   su pokybuild -c "yocto-autobuilder-helper/janitor/ab-janitor" &
   su pokybuild -c "buildbot start yocto-controller"
   #tail -F yocto-controller/twistd.log &
-elif [ "${role}" = "worker" ]; then
-  workername="$2"
-  su pokybuild -c "buildbot-worker start ${workername}"
+elif [ "${role}" = "worker" ] || [ "${role}" = "extraworker" ]; then
+  if [ "${role}" = "extraworker" ]; then
+      worker_name="local-worker-extra-${instance}"
+  else
+      worker_name=local-worker-debian
+  fi
+  buildbot-worker create-worker -r --umask=0o22 yocto-worker controller "${worker_name}" pass
+  chown -R pokybuild:nogroup yocto-worker
+  su pokybuild -c "buildbot-worker start yocto-worker"
 else
   echo "Unexpected role: ${role}"
   exit 2