MIRRORS/yocto-autobuilder-helper
1
0
Fork 0
mirror of git://git.yoctoproject.org/yocto-autobuilder-helper.git synced 2025-10-22 23:02:21 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

shared-repos: Use tar instead of rsync for speed

Browse Source 
The rysnc of 20,000 files (650MB) onto the nas is slow taking ~3 minutes
at idle and worse at load. This is due to the number of files which
is a pain point for NFS. This piece of the build is also a bottleneck
since the rest of a build depends on it happening.

If we switch to zstd compressed tar, it takes 2.49s. Other compression
methods were much slower but zstd seems 'accptable' and speeds things
up too.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Richard Purdie 2021-10-30 23:01:59 +01:00
parent 5dc0559cc8
commit aff49e938e
3 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/prepare-shared-repos
View File

@ -39,5 +39,5 @@ with tempfile.TemporaryDirectory(prefix="shared-repo-temp-", dir="/home/pokybuil
if args.publish_dir:
utils.publishrepo(tempdir, repo, args.publish_dir)
utils.printheader("Running rsync")
subprocess.check_call("rsync -a " + tempdir + "/* " + args.sharedsrcdir, shell=True)
utils.printheader("Creating shared src tarball")
subprocess.check_call("tar -I zstd -cf " + args.sharedsrcdir.rstrip("/") + ".tar.zst ./*", shell=True, cwd=tempdir)

6
scripts/send-qa-email
View File

@ -45,9 +45,11 @@ buildtoolsdir = os.path.dirname(args.repojson) + "/build/buildtools"
if os.path.exists(buildtoolsdir):
utils.enable_buildtools_tarball(buildtoolsdir)
repodir = os.path.dirname(args.repojson) + "/build/repos"
if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
# Need the finalised revisions (not 'HEAD')
targetrepodir = "%s/poky" % (args.sharedrepodir)
targetrepodir = "%s/poky" % (repodir)
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=targetrepodir).decode('utf-8').strip()
branch = repos['poky']['branch']
repo = repos['poky']['url']
@ -116,7 +118,7 @@ if args.send.lower() != 'true' or not args.publish_dir or not args.release:
buildhashes = ""
for repo in sorted(repos.keys()):
# Need the finalised revisions (not 'HEAD')
targetrepodir = "%s/%s" % (args.sharedrepodir, repo)
targetrepodir = "%s/%s" % (repodir, repo)
revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=targetrepodir).decode('utf-8').strip()
buildhashes += "%s: %s\n" % (repo, revision)

9
scripts/shared-repo-unpack
View File

@ -50,11 +50,14 @@ needrepos_baseddirs = [r.split('/')[0] for r in needrepos]
for repo in sorted(repos.keys()):
if repo not in needrepos_baseddirs:
continue
targetrepodir = "%s/%s" % (targetsubdir, repo)
if args.cache_dir:
utils.printheader("Copying in repo %s" % repo)
utils.mkdir(targetrepodir)
subprocess.check_call(["rsync", "-a", "%s/%s" % (args.cache_dir, repo), targetsubdir])
utils.mkdir(targetsubdir)
if args.target in ["a-full", "a-quick"]:
# full/quick need all repo data due to send-qa-email
subprocess.check_call(["tar", "-I", "zstd", "-C", targetsubdir, "-xf", "%s.tar.zst" % args.cache_dir])
else:
subprocess.check_call(["tar", "-I", "zstd", "-C", targetsubdir, "-xf", "%s.tar.zst" % args.cache_dir, "./" + repo])
else:
utils.printheader("Fetching repo %s" % repo)
utils.fetchgitrepo(targetsubdir, repo, repos[repo], stashdir)