MIRRORS/yocto-autobuilder2
1
0
Fork 0
mirror of git://git.yoctoproject.org/yocto-autobuilder2.git synced 2025-07-19 20:59:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

builders: Improve disk space checking

Browse Source 
We've been seeing increasing problems with disk space issues breaking builds.

Increase the check from 100GB to 200GB
Monitor /tmp for 10GB free as well
Refactor the code to make adding other mount point checks easy

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2024-08-04 12:57:41 +01:00
parent 694d2a9bae
commit d809e29a23
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
builders.py
View File

@ -65,15 +65,24 @@ def shell(command, worker, builder):
def canStartBuild(builder, wfb, request):
log.msg("Checking available disk space...")
cmd = yield shell("df -BG | grep $(findmnt -T . | awk '{print $2}' | sed -n 2p) | awk '{print $4}' | sed 's/[^0-9]*//g'", wfb.worker, builder)
threshold = 100 # GB of space
# threshold is GB of space
checks = {
"." : (200, "HOME"),
"/tmp" : (10, "/tmp"),
}
for mountpoint in checks:
threshold, name = checks[mountpoint]
cmd = yield shell("df -BG | grep $(findmnt -T %s | awk '{print $2}' | sed -n 2p) | awk '{print $4}' | sed 's/[^0-9]*//g'" % mountpoint, wfb.worker, builder)
if int(cmd.stdout) < threshold:
log.msg("Detected {0} GB of space available, less than threshold of {1} GB. Can't start build".format(cmd.stdout, threshold))
log.msg("Detected {0} GB of space available on {1}, less than threshold of {2} GB. Can't start build".format(cmd.stdout, name, threshold))
wfb.worker.quarantine_timeout = 10 * 60
wfb.worker.putInQuarantine()
return False
log.msg("Detected {0} GB of space available for {1}, more than threshold of {2} GB".format(cmd.stdout, name, threshold))
log.msg("Detected {0} GB of space available, more than threshold of {1} GB. OK to build".format(cmd.stdout, threshold))
log.msg("OK to build")
if wfb.worker.isPaused():
# It was low on space so delay more than one build starting for a while
wfb.worker.quarantine_timeout = 15 * 60