builders: Tweak space limits

Practise is showing 60GB can still let the builders run out of space. Increase
to 100GB and also delay things more to allow better recovery, 2 minutes isn't
a long time when dealing with file deletion.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2021-11-12 12:11:55 +00:00
parent 8829a85464
commit de5263b7be

View File

@ -66,17 +66,17 @@ def canStartBuild(builder, wfb, request):
log.msg("Checking available disk space...")
cmd = yield shell("df -BG | grep $(findmnt -T . | awk '{print $2}' | sed -n 2p) | awk '{print $4}' | sed 's/[^0-9]*//g'", wfb.worker, builder)
threshold = 60 # GB of space
threshold = 100 # GB of space
if int(cmd.stdout) < threshold:
log.msg("Detected {0} GB of space available, less than threshold of {1} GB. Can't start build".format(cmd.stdout, threshold))
wfb.worker.quarantine_timeout = 2 * 60
wfb.worker.quarantine_timeout = 10 * 60
wfb.worker.putInQuarantine()
return False
log.msg("Detected {0} GB of space available, more than threshold of {1} GB. OK to build".format(cmd.stdout, threshold))
if wfb.worker.isPaused():
# It was low on space so delay the builds starting a bit
wfb.worker.quarantine_timeout = 2 * 60
# It was low on space so delay more than one build starting for a while
wfb.worker.quarantine_timeout = 15 * 60
wfb.worker.putInQuarantine()
else:
wfb.worker.exitQuarantine()