mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
c3b3a681f7
If you specify one or more components to update immediately following the "update" command, only these components will be updated as opposed to the default behaviour of updating all of them. (From OE-Core rev: 1779afce024b31edcf846329da705ed5aa9fe63e) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
385 lines
14 KiB
Python
Executable File
385 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
#
|
|
# Copyright 2011 Intel Corporation
|
|
# Authored-by: Yu Ke <ke.yu@intel.com>
|
|
# Paul Eggleton <paul.eggleton@intel.com>
|
|
# Richard Purdie <richard.purdie@intel.com>
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import os, sys
|
|
import optparse
|
|
import logging
|
|
import subprocess
|
|
import ConfigParser
|
|
|
|
__version__ = "0.2.1"
|
|
|
|
def logger_create():
|
|
logger = logging.getLogger("")
|
|
loggerhandler = logging.StreamHandler()
|
|
loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
|
|
logger.addHandler(loggerhandler)
|
|
logger.setLevel(logging.INFO)
|
|
return logger
|
|
|
|
logger = logger_create()
|
|
|
|
class Configuration(object):
|
|
"""
|
|
Manages the configuration
|
|
|
|
For an example config file, see combo-layer.conf.example
|
|
|
|
"""
|
|
def __init__(self, options):
|
|
for key, val in options.__dict__.items():
|
|
setattr(self, key, val)
|
|
self.parser = ConfigParser.ConfigParser()
|
|
self.parser.readfp(open(self.conffile))
|
|
self.repos = {}
|
|
for repo in self.parser.sections():
|
|
self.repos[repo] = {}
|
|
for (name, value) in self.parser.items(repo):
|
|
if value.startswith("@"):
|
|
self.repos[repo][name] = eval(value.strip("@"))
|
|
else:
|
|
self.repos[repo][name] = value
|
|
|
|
def update(self, repo, option, value):
|
|
self.parser.set(repo, option, value)
|
|
self.parser.write(open(self.conffile, "w"))
|
|
|
|
def sanity_check(self):
|
|
required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
|
|
msg = ""
|
|
for name in self.repos:
|
|
for option in required_options:
|
|
if option not in self.repos[name]:
|
|
msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
|
|
if msg != "":
|
|
logger.error("configuration file %s has the following error: %s" % (self.conffile,msg))
|
|
sys.exit(1)
|
|
|
|
# filterdiff is required by action_splitpatch, so check its availability
|
|
if subprocess.call("which filterdiff &>/dev/null", shell=True) != 0:
|
|
logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
|
|
sys.exit(1)
|
|
|
|
def runcmd(cmd,destdir=None,printerr=True):
|
|
"""
|
|
execute command, raise CalledProcessError if fail
|
|
return output if succeed
|
|
"""
|
|
logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
|
|
out = os.tmpfile()
|
|
try:
|
|
subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
|
|
except subprocess.CalledProcessError,e:
|
|
out.seek(0)
|
|
if printerr:
|
|
logger.error("%s" % out.read())
|
|
raise e
|
|
|
|
out.seek(0)
|
|
output = out.read()
|
|
logger.debug("output: %s" % output )
|
|
return output
|
|
|
|
def action_init(conf, args):
|
|
"""
|
|
Clone component repositories
|
|
Check git is initialised; if not, copy initial data from component repos
|
|
"""
|
|
for name in conf.repos:
|
|
ldir = conf.repos[name]['local_repo_dir']
|
|
if not os.path.exists(ldir):
|
|
logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
|
|
subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
|
|
branch = conf.repos[name].get('branch', "master")
|
|
runcmd("git checkout %s" % branch, ldir)
|
|
if not os.path.exists(".git"):
|
|
runcmd("git init")
|
|
for name in conf.repos:
|
|
repo = conf.repos[name]
|
|
ldir = repo['local_repo_dir']
|
|
logger.info("copying data from %s..." % name)
|
|
dest_dir = repo['dest_dir']
|
|
if dest_dir and dest_dir != ".":
|
|
extract_dir = os.path.join(os.getcwd(), dest_dir)
|
|
os.makedirs(extract_dir)
|
|
else:
|
|
extract_dir = os.getcwd()
|
|
branch = repo.get('branch', "master")
|
|
file_filter = repo.get('file_filter', "")
|
|
runcmd("git archive %s | tar -x -C %s %s" % (branch, extract_dir, file_filter), ldir)
|
|
lastrev = runcmd("git rev-parse HEAD", ldir).strip()
|
|
conf.update(name, "last_revision", lastrev)
|
|
runcmd("git add .")
|
|
logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.")
|
|
else:
|
|
logger.info("Repository already initialised, nothing to do.")
|
|
|
|
|
|
def check_repo_clean(repodir):
|
|
"""
|
|
check if the repo is clean
|
|
exit if repo is dirty
|
|
"""
|
|
output=runcmd("git status --porcelain", repodir)
|
|
if output:
|
|
logger.error("git repo %s is dirty, please fix it first", repodir)
|
|
sys.exit(1)
|
|
|
|
def action_update(conf, args):
|
|
"""
|
|
update the component repos
|
|
generate the patch list
|
|
apply the generated patches
|
|
"""
|
|
# make sure all repos are clean
|
|
for name in conf.repos:
|
|
check_repo_clean(conf.repos[name]['local_repo_dir'])
|
|
check_repo_clean(os.getcwd())
|
|
|
|
import uuid
|
|
patch_dir = "patch-%s" % uuid.uuid4()
|
|
os.mkdir(patch_dir)
|
|
|
|
repos = []
|
|
if len(args) > 1:
|
|
for arg in args[1:]:
|
|
if arg.startswith('-'):
|
|
break
|
|
else:
|
|
repos.append(arg)
|
|
for repo in repos:
|
|
if not repo in conf.repos:
|
|
logger.error("Specified component '%s' not found in configuration" % repo)
|
|
sys.exit(0)
|
|
|
|
if not repos:
|
|
repos = conf.repos
|
|
|
|
for name in repos:
|
|
repo = conf.repos[name]
|
|
ldir = repo['local_repo_dir']
|
|
dest_dir = repo['dest_dir']
|
|
branch = repo.get('branch', "master")
|
|
repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
|
|
|
|
# Step 1: update the component repo
|
|
runcmd("git checkout %s" % branch, ldir)
|
|
logger.info("git pull for component repo %s in %s ..." % (name, ldir))
|
|
output=runcmd("git pull", ldir)
|
|
logger.info(output)
|
|
|
|
# Step 2: generate the patch list and store to patch dir
|
|
logger.info("generating patches for %s" % name)
|
|
if dest_dir != ".":
|
|
prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
|
|
else:
|
|
prefix = ""
|
|
if repo['last_revision'] == "":
|
|
logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
|
|
patch_cmd_range = "--root %s" % branch
|
|
rev_cmd_range = branch
|
|
else:
|
|
patch_cmd_range = "%s..%s" % (repo['last_revision'], branch)
|
|
rev_cmd_range = patch_cmd_range
|
|
|
|
file_filter = repo.get('file_filter',"")
|
|
|
|
patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
|
|
(prefix,repo_patch_dir, patch_cmd_range, file_filter)
|
|
output = runcmd(patch_cmd, ldir)
|
|
logger.debug("generated patch set:\n%s" % output)
|
|
patchlist = output.splitlines()
|
|
|
|
rev_cmd = 'git rev-list --no-merges ' + rev_cmd_range
|
|
revlist = runcmd(rev_cmd, ldir).splitlines()
|
|
|
|
# Step 3: Call repo specific hook to adjust patch
|
|
if 'hook' in repo:
|
|
# hook parameter is: ./hook patchpath revision reponame
|
|
count=len(revlist)-1
|
|
for patch in patchlist:
|
|
runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
|
|
count=count-1
|
|
|
|
# Step 4: write patch list and revision list to file, for user to edit later
|
|
patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
|
|
repo['patchlist'] = patchlist_file
|
|
f = open(patchlist_file, 'w')
|
|
count=len(revlist)-1
|
|
for patch in patchlist:
|
|
f.write("%s %s\n" % (patch, revlist[count]))
|
|
count=count-1
|
|
f.close()
|
|
|
|
# Step 5: invoke bash for user to edit patch and patch list
|
|
if conf.interactive:
|
|
print 'Edit the patch and patch list in %s\n' \
|
|
'For example, remove the unwanted patch entry from patchlist-*, so that it will be not applied later\n' \
|
|
'When you are finished, run the following to continue:\n' \
|
|
' exit 0 -- exit and continue to apply the patch\n' \
|
|
' exit 1 -- abort and do not apply the patch\n' % patch_dir
|
|
ret = subprocess.call(["bash"], cwd=patch_dir)
|
|
if ret != 0:
|
|
print "Aborting without applying the patch"
|
|
sys.exit(0)
|
|
|
|
# Step 6: apply the generated and revised patch
|
|
apply_patchlist(conf, repos)
|
|
runcmd("rm -rf %s" % patch_dir)
|
|
|
|
# Step 7: commit the updated config file if it's being tracked
|
|
relpath = os.path.relpath(conf.conffile)
|
|
try:
|
|
output = runcmd("git status --porcelain %s" % relpath, printerr=False)
|
|
except:
|
|
# Outside the repository
|
|
output = None
|
|
if output:
|
|
logger.info("Committing updated configuration file")
|
|
if output.lstrip().startswith("M"):
|
|
runcmd('git commit -m "Automatic commit to update last_revision" %s' % relpath)
|
|
|
|
def apply_patchlist(conf, repos):
|
|
"""
|
|
apply the generated patch list to combo repo
|
|
"""
|
|
for name in repos:
|
|
repo = conf.repos[name]
|
|
lastrev = repo["last_revision"]
|
|
for line in open(repo['patchlist']):
|
|
patchfile = line.split()[0]
|
|
lastrev = line.split()[1]
|
|
if os.path.getsize(patchfile) == 0:
|
|
logger.info("(skipping %s - no changes)", lastrev)
|
|
else:
|
|
cmd = "git am --keep-cr -s -p1 %s" % patchfile
|
|
logger.info("Apply %s" % patchfile )
|
|
try:
|
|
runcmd(cmd)
|
|
except subprocess.CalledProcessError:
|
|
logger.info('running "git am --abort" to cleanup repo')
|
|
runcmd("git am --abort")
|
|
logger.error('"%s" failed' % cmd)
|
|
logger.info("please manually apply patch %s" % patchfile)
|
|
logger.info("After applying, run this tool again to apply the remaining patches")
|
|
conf.update(name, "last_revision", lastrev)
|
|
sys.exit(0)
|
|
if lastrev != repo['last_revision']:
|
|
conf.update(name, "last_revision", lastrev)
|
|
|
|
def action_splitpatch(conf, args):
|
|
"""
|
|
generate the commit patch and
|
|
split the patch per repo
|
|
"""
|
|
logger.debug("action_splitpatch")
|
|
if len(args) > 1:
|
|
commit = args[1]
|
|
else:
|
|
commit = "HEAD"
|
|
patchdir = "splitpatch-%s" % commit
|
|
if not os.path.exists(patchdir):
|
|
os.mkdir(patchdir)
|
|
|
|
# filerange_root is for the repo whose dest_dir is root "."
|
|
# and it should be specified by excluding all other repo dest dir
|
|
# like "-x repo1 -x repo2 -x repo3 ..."
|
|
filerange_root = ""
|
|
for name in conf.repos:
|
|
dest_dir = conf.repos[name]['dest_dir']
|
|
if dest_dir != ".":
|
|
filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
|
|
|
|
for name in conf.repos:
|
|
dest_dir = conf.repos[name]['dest_dir']
|
|
patch_filename = "%s/%s.patch" % (patchdir, name)
|
|
if dest_dir == ".":
|
|
cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
|
|
else:
|
|
cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
|
|
runcmd(cmd)
|
|
# Detect empty patches (including those produced by filterdiff above
|
|
# that contain only preamble text)
|
|
if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
|
|
os.remove(patch_filename)
|
|
logger.info("(skipping %s - no changes)", name)
|
|
else:
|
|
logger.info(patch_filename)
|
|
|
|
def action_error(conf, args):
|
|
logger.info("invalid action %s" % args[0])
|
|
|
|
actions = {
|
|
"init": action_init,
|
|
"update": action_update,
|
|
"splitpatch": action_splitpatch,
|
|
}
|
|
|
|
def main():
|
|
parser = optparse.OptionParser(
|
|
version = "Combo Layer Repo Tool version %s" % __version__,
|
|
usage = """%prog [options] action
|
|
|
|
Create and update a combination layer repository from multiple component repositories.
|
|
|
|
Action:
|
|
init initialise the combo layer repo
|
|
update [components] get patches from component repos and apply them to the combo repo
|
|
splitpatch [commit] generate commit patch and split per component, default commit is HEAD""")
|
|
|
|
parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).",
|
|
action = "store", dest = "conffile", default = "conf/combo-layer.conf")
|
|
|
|
parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches",
|
|
action = "store_true", dest = "interactive", default = False)
|
|
|
|
parser.add_option("-D", "--debug", help = "output debug information",
|
|
action = "store_true", dest = "debug", default = False)
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
# Dispatch to action handler
|
|
if len(args) == 1:
|
|
logger.error("No action specified, exiting")
|
|
parser.print_help()
|
|
elif args[1] not in actions:
|
|
logger.error("Unsupported action %s, exiting\n" % (args[1]))
|
|
parser.print_help()
|
|
elif not os.path.exists(options.conffile):
|
|
logger.error("No valid config file, exiting\n")
|
|
parser.print_help()
|
|
else:
|
|
if options.debug:
|
|
logger.setLevel(logging.DEBUG)
|
|
confdata = Configuration(options)
|
|
confdata.sanity_check()
|
|
actions.get(args[1], action_error)(confdata, args[1:])
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
ret = main()
|
|
except Exception:
|
|
ret = 1
|
|
import traceback
|
|
traceback.print_exc(5)
|
|
sys.exit(ret)
|