mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-06 21:54:46 +02:00
c2813a4c69
Make the deploy function independent from d. This allows to call the function also from Python code not running in bitbake. This is needed to for the devtool ide plugin which will call the do_install task and the code from devtool deploy-target independently from a bitbake server. This allows a much quicker workflow. (From OE-Core rev: c8697d1132cbd4b2a2502b4c48e7d91fc18de786) Signed-off-by: Adrian Freihofer <adrian.freihofer@siemens.com> Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
379 lines
18 KiB
Python
379 lines
18 KiB
Python
# Development tool - deploy/undeploy command plugin
|
|
#
|
|
# Copyright (C) 2014-2016 Intel Corporation
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
"""Devtool plugin containing the deploy subcommands"""
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
|
|
import bb.utils
|
|
import argparse_oe
|
|
import oe.types
|
|
|
|
from devtool import exec_fakeroot_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError
|
|
|
|
logger = logging.getLogger('devtool')
|
|
|
|
deploylist_path = '/.devtool'
|
|
|
|
def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
|
|
"""
|
|
Prepare a shell script for running on the target to
|
|
deploy/undeploy files. We have to be careful what we put in this
|
|
script - only commands that are likely to be available on the
|
|
target are suitable (the target might be constrained, e.g. using
|
|
busybox rather than bash with coreutils).
|
|
"""
|
|
lines = []
|
|
lines.append('#!/bin/sh')
|
|
lines.append('set -e')
|
|
if undeployall:
|
|
# Yes, I know this is crude - but it does work
|
|
lines.append('for entry in %s/*.list; do' % deploylist_path)
|
|
lines.append('[ ! -f $entry ] && exit')
|
|
lines.append('set `basename $entry | sed "s/.list//"`')
|
|
if dryrun:
|
|
if not deploy:
|
|
lines.append('echo "Previously deployed files for $1:"')
|
|
lines.append('manifest="%s/$1.list"' % deploylist_path)
|
|
lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
|
|
lines.append('if [ -f $manifest ] ; then')
|
|
# Read manifest in reverse and delete files / remove empty dirs
|
|
lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
|
|
lines.append(' do')
|
|
if dryrun:
|
|
lines.append(' if [ ! -d $file ] ; then')
|
|
lines.append(' echo $file')
|
|
lines.append(' fi')
|
|
else:
|
|
lines.append(' if [ -d $file ] ; then')
|
|
# Avoid deleting a preserved directory in case it has special perms
|
|
lines.append(' if [ ! -d $preservedir/$file ] ; then')
|
|
lines.append(' rmdir $file > /dev/null 2>&1 || true')
|
|
lines.append(' fi')
|
|
lines.append(' else')
|
|
lines.append(' rm -f $file')
|
|
lines.append(' fi')
|
|
lines.append(' done')
|
|
if not dryrun:
|
|
lines.append(' rm $manifest')
|
|
if not deploy and not dryrun:
|
|
# May as well remove all traces
|
|
lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true')
|
|
lines.append('fi')
|
|
|
|
if deploy:
|
|
if not nocheckspace:
|
|
# Check for available space
|
|
# FIXME This doesn't take into account files spread across multiple
|
|
# partitions, but doing that is non-trivial
|
|
# Find the part of the destination path that exists
|
|
lines.append('checkpath="$2"')
|
|
lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
|
|
lines.append('do')
|
|
lines.append(' checkpath=`dirname "$checkpath"`')
|
|
lines.append('done')
|
|
lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
|
|
# First line of the file is the total space
|
|
lines.append('total=`head -n1 $3`')
|
|
lines.append('if [ $total -gt $freespace ] ; then')
|
|
lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
|
|
lines.append(' exit 1')
|
|
lines.append('fi')
|
|
if not nopreserve:
|
|
# Preserve any files that exist. Note that this will add to the
|
|
# preserved list with successive deployments if the list of files
|
|
# deployed changes, but because we've deleted any previously
|
|
# deployed files at this point it will never preserve anything
|
|
# that was deployed, only files that existed prior to any deploying
|
|
# (which makes the most sense)
|
|
lines.append('cat $3 | sed "1d" | while read file fsize')
|
|
lines.append('do')
|
|
lines.append(' if [ -e $file ] ; then')
|
|
lines.append(' dest="$preservedir/$file"')
|
|
lines.append(' mkdir -p `dirname $dest`')
|
|
lines.append(' mv $file $dest')
|
|
lines.append(' fi')
|
|
lines.append('done')
|
|
lines.append('rm $3')
|
|
lines.append('mkdir -p `dirname $manifest`')
|
|
lines.append('mkdir -p $2')
|
|
if verbose:
|
|
lines.append(' tar xv -C $2 -f - | tee $manifest')
|
|
else:
|
|
lines.append(' tar xv -C $2 -f - > $manifest')
|
|
lines.append('sed -i "s!^./!$2!" $manifest')
|
|
elif not dryrun:
|
|
# Put any preserved files back
|
|
lines.append('if [ -d $preservedir ] ; then')
|
|
lines.append(' cd $preservedir')
|
|
# find from busybox might not have -exec, so we don't use that
|
|
lines.append(' find . -type f | while read file')
|
|
lines.append(' do')
|
|
lines.append(' mv $file /$file')
|
|
lines.append(' done')
|
|
lines.append(' cd /')
|
|
lines.append(' rm -rf $preservedir')
|
|
lines.append('fi')
|
|
|
|
if undeployall:
|
|
if not dryrun:
|
|
lines.append('echo "NOTE: Successfully undeployed $1"')
|
|
lines.append('done')
|
|
|
|
# Delete the script itself
|
|
lines.append('rm $0')
|
|
lines.append('')
|
|
|
|
return '\n'.join(lines)
|
|
|
|
def deploy(args, config, basepath, workspace):
|
|
"""Entry point for the devtool 'deploy' subcommand"""
|
|
import oe.utils
|
|
|
|
check_workspace_recipe(workspace, args.recipename, checksrc=False)
|
|
|
|
tinfoil = setup_tinfoil(basepath=basepath)
|
|
try:
|
|
try:
|
|
rd = tinfoil.parse_recipe(args.recipename)
|
|
except Exception as e:
|
|
raise DevtoolError('Exception parsing recipe %s: %s' %
|
|
(args.recipename, e))
|
|
|
|
srcdir = rd.getVar('D')
|
|
workdir = rd.getVar('WORKDIR')
|
|
path = rd.getVar('PATH')
|
|
strip_cmd = rd.getVar('STRIP')
|
|
libdir = rd.getVar('libdir')
|
|
base_libdir = rd.getVar('base_libdir')
|
|
max_process = oe.utils.get_bb_number_threads(rd)
|
|
fakerootcmd = rd.getVar('FAKEROOTCMD')
|
|
fakerootenv = rd.getVar('FAKEROOTENV')
|
|
finally:
|
|
tinfoil.shutdown()
|
|
|
|
return deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args)
|
|
|
|
def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args):
|
|
import math
|
|
import oe.package
|
|
|
|
try:
|
|
host, destdir = args.target.split(':')
|
|
except ValueError:
|
|
destdir = '/'
|
|
else:
|
|
args.target = host
|
|
if not destdir.endswith('/'):
|
|
destdir += '/'
|
|
|
|
recipe_outdir = srcdir
|
|
if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
|
|
raise DevtoolError('No files to deploy - have you built the %s '
|
|
'recipe? If so, the install step has not installed '
|
|
'any files.' % args.recipename)
|
|
|
|
if args.strip and not args.dry_run:
|
|
# Fakeroot copy to new destination
|
|
srcdir = recipe_outdir
|
|
recipe_outdir = os.path.join(workdir, 'devtool-deploy-target-stripped')
|
|
if os.path.isdir(recipe_outdir):
|
|
exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True)
|
|
exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
|
|
os.environ['PATH'] = ':'.join([os.environ['PATH'], path or ''])
|
|
oe.package.strip_execs(args.recipename, recipe_outdir, strip_cmd, libdir, base_libdir, max_process)
|
|
|
|
filelist = []
|
|
inodes = set({})
|
|
ftotalsize = 0
|
|
for root, _, files in os.walk(recipe_outdir):
|
|
for fn in files:
|
|
fstat = os.lstat(os.path.join(root, fn))
|
|
# Get the size in kiB (since we'll be comparing it to the output of du -k)
|
|
# MUST use lstat() here not stat() or getfilesize() since we don't want to
|
|
# dereference symlinks
|
|
if fstat.st_ino in inodes:
|
|
fsize = 0
|
|
else:
|
|
fsize = int(math.ceil(float(fstat.st_size)/1024))
|
|
inodes.add(fstat.st_ino)
|
|
ftotalsize += fsize
|
|
# The path as it would appear on the target
|
|
fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
|
|
filelist.append((fpath, fsize))
|
|
|
|
if args.dry_run:
|
|
print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
|
|
for item, _ in filelist:
|
|
print(' %s' % item)
|
|
return 0
|
|
|
|
extraoptions = ''
|
|
if args.no_host_check:
|
|
extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
|
|
if not args.show_status:
|
|
extraoptions += ' -q'
|
|
|
|
scp_sshexec = ''
|
|
ssh_sshexec = 'ssh'
|
|
if args.ssh_exec:
|
|
scp_sshexec = "-S %s" % args.ssh_exec
|
|
ssh_sshexec = args.ssh_exec
|
|
scp_port = ''
|
|
ssh_port = ''
|
|
if args.port:
|
|
scp_port = "-P %s" % args.port
|
|
ssh_port = "-p %s" % args.port
|
|
|
|
if args.key:
|
|
extraoptions += ' -i %s' % args.key
|
|
|
|
# In order to delete previously deployed files and have the manifest file on
|
|
# the target, we write out a shell script and then copy it to the target
|
|
# so we can then run it (piping tar output to it).
|
|
# (We cannot use scp here, because it doesn't preserve symlinks.)
|
|
tmpdir = tempfile.mkdtemp(prefix='devtool')
|
|
try:
|
|
tmpscript = '/tmp/devtool_deploy.sh'
|
|
tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
|
|
shellscript = _prepare_remote_script(deploy=True,
|
|
verbose=args.show_status,
|
|
nopreserve=args.no_preserve,
|
|
nocheckspace=args.no_check_space)
|
|
# Write out the script to a file
|
|
with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
|
|
f.write(shellscript)
|
|
# Write out the file list
|
|
with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
|
|
f.write('%d\n' % ftotalsize)
|
|
for fpath, fsize in filelist:
|
|
f.write('%s %d\n' % (fpath, fsize))
|
|
# Copy them to the target
|
|
ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
|
|
if ret != 0:
|
|
raise DevtoolError('Failed to copy script to %s - rerun with -s to '
|
|
'get a complete error message' % args.target)
|
|
finally:
|
|
shutil.rmtree(tmpdir)
|
|
|
|
# Now run the script
|
|
ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
|
|
if ret != 0:
|
|
raise DevtoolError('Deploy failed - rerun with -s to get a complete '
|
|
'error message')
|
|
|
|
logger.info('Successfully deployed %s' % recipe_outdir)
|
|
|
|
files_list = []
|
|
for root, _, files in os.walk(recipe_outdir):
|
|
for filename in files:
|
|
filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
|
|
files_list.append(os.path.join(destdir, filename))
|
|
|
|
return 0
|
|
|
|
def undeploy(args, config, basepath, workspace):
|
|
"""Entry point for the devtool 'undeploy' subcommand"""
|
|
if args.all and args.recipename:
|
|
raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
|
|
elif not args.recipename and not args.all:
|
|
raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
|
|
|
|
extraoptions = ''
|
|
if args.no_host_check:
|
|
extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
|
|
if not args.show_status:
|
|
extraoptions += ' -q'
|
|
|
|
scp_sshexec = ''
|
|
ssh_sshexec = 'ssh'
|
|
if args.ssh_exec:
|
|
scp_sshexec = "-S %s" % args.ssh_exec
|
|
ssh_sshexec = args.ssh_exec
|
|
scp_port = ''
|
|
ssh_port = ''
|
|
if args.port:
|
|
scp_port = "-P %s" % args.port
|
|
ssh_port = "-p %s" % args.port
|
|
|
|
args.target = args.target.split(':')[0]
|
|
|
|
tmpdir = tempfile.mkdtemp(prefix='devtool')
|
|
try:
|
|
tmpscript = '/tmp/devtool_undeploy.sh'
|
|
shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
|
|
# Write out the script to a file
|
|
with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
|
|
f.write(shellscript)
|
|
# Copy it to the target
|
|
ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
|
|
if ret != 0:
|
|
raise DevtoolError('Failed to copy script to %s - rerun with -s to '
|
|
'get a complete error message' % args.target)
|
|
finally:
|
|
shutil.rmtree(tmpdir)
|
|
|
|
# Now run the script
|
|
ret = subprocess.call('%s %s %s %s \'sh %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
|
|
if ret != 0:
|
|
raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
|
|
'error message')
|
|
|
|
if not args.all and not args.dry_run:
|
|
logger.info('Successfully undeployed %s' % args.recipename)
|
|
return 0
|
|
|
|
|
|
def register_commands(subparsers, context):
|
|
"""Register devtool subcommands from the deploy plugin"""
|
|
|
|
parser_deploy = subparsers.add_parser('deploy-target',
|
|
help='Deploy recipe output files to live target machine',
|
|
description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
|
|
group='testbuild')
|
|
parser_deploy.add_argument('recipename', help='Recipe to deploy')
|
|
parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
|
|
parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
|
|
parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
|
|
parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
|
|
parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
|
|
parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
|
|
parser_deploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
|
|
parser_deploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
|
|
parser_deploy.add_argument('-I', '--key',
|
|
help='Specify ssh private key for connection to the target')
|
|
|
|
strip_opts = parser_deploy.add_mutually_exclusive_group(required=False)
|
|
strip_opts.add_argument('-S', '--strip',
|
|
help='Strip executables prior to deploying (default: %(default)s). '
|
|
'The default value of this option can be controlled by setting the strip option in the [Deploy] section to True or False.',
|
|
default=oe.types.boolean(context.config.get('Deploy', 'strip', default='0')),
|
|
action='store_true')
|
|
strip_opts.add_argument('--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
|
|
|
|
parser_deploy.set_defaults(func=deploy)
|
|
|
|
parser_undeploy = subparsers.add_parser('undeploy-target',
|
|
help='Undeploy recipe output files in live target machine',
|
|
description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
|
|
group='testbuild')
|
|
parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
|
|
parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
|
|
parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
|
|
parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
|
|
parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
|
|
parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
|
|
parser_undeploy.add_argument('-e', '--ssh-exec', help='Executable to use in place of ssh')
|
|
parser_undeploy.add_argument('-P', '--port', help='Specify port to use for connection to the target')
|
|
parser_undeploy.add_argument('-I', '--key',
|
|
help='Specify ssh private key for connection to the target')
|
|
|
|
parser_undeploy.set_defaults(func=undeploy)
|