update: fix logic for updating layer dependencies at the end

* We were passing the incorrect path (to the top of the layer repo) if
  the layer had a subdirectory, so this doesn't seem to have been able
  to work for such layers previously.
* Doing this update in the main update.py script meant that this could
  never work across branches requiring a python version change (using
  PythonEnvironment records) since the code was running within the same
  environment in which update.py was launched - the entire point of the
  separation of the two scripts. Move the checking to update_layer.py
  and call it separately to perform these updates, splitting out some
  common code in order to do so.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

layerindex/update.py

@@ -17,7 +17,6 @@ import signal
 from datetime import datetime, timedelta
 from distutils.version import LooseVersion
 import utils
-from layerconfparse import LayerConfParse
 
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -62,6 +61,30 @@ def run_command_interruptible(cmd):
     return process.returncode, buf
 
 
+def prepare_update_layer_command(options, branch, layer, updatedeps=False):
+    """Prepare the update_layer.py command line"""
+    if branch.update_environment:
+        cmdprefix = branch.update_environment.get_command()
+    else:
+        cmdprefix = 'python3'
+    cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name)
+    if updatedeps:
+        cmd += ' --update-dependencies'
+    if options.reload:
+        cmd += ' --reload'
+    if options.fullreload:
+        cmd += ' --fullreload'
+    if options.nocheckout:
+        cmd += ' --nocheckout'
+    if options.dryrun:
+        cmd += ' -n'
+    if options.loglevel == logging.DEBUG:
+        cmd += ' -d'
+    elif options.loglevel == logging.ERROR:
+        cmd += ' -q'
+    return cmd
+
+
 def main():
     if LooseVersion(git.__version__) < '0.3.1':
         logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
@@ -221,24 +244,7 @@ def main():
                     urldir = layer.get_fetch_dir()
                     repodir = os.path.join(fetchdir, urldir)
 
-                    if branchobj.update_environment:
+                    cmd = prepare_update_layer_command(options, branchobj, layer)
-                        cmdprefix = branchobj.update_environment.get_command()
-                    else:
-                        cmdprefix = 'python3'
-                    cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch)
-                    if options.reload:
-                        cmd += ' --reload'
-                    if options.fullreload:
-                        cmd += ' --fullreload'
-                    if options.nocheckout:
-                        cmd += ' --nocheckout'
-                    if options.dryrun:
-                        cmd += ' -n'
-                    if options.loglevel == logging.DEBUG:
-                        cmd += ' -d'
-                    elif options.loglevel == logging.ERROR:
-                        cmd += ' -q'
-
                     logger.debug('Running layer update command: %s' % cmd)
                     layerupdate.started = datetime.now()
                     ret, output = run_command_interruptible(cmd)
@@ -265,32 +271,23 @@ def main():
             # dependencies that may have been missed.  Note that creating the
             # dependencies is a best-effort and continues if they are not found.
             for branch in branches:
-                layerconfparser = LayerConfParse(logger=logger, bitbakepath=bitbakepath)
+                branchobj = utils.get_branch(branch)
-                try:
+                for layer in layerquery:
-                    for layer in layerquery:
+                    layerbranch = layer.get_layerbranch(branch)
-
+                    if layerbranch:
-                        layerbranch = layer.get_layerbranch(branch)
+                        if not (options.reload or options.fullreload):
-                        # Skip layers that did not change.
+                            # Skip layers that did not change.
-                        layer_last_rev = None
-                        if layerbranch:
                             layer_last_rev = last_rev.get(layerbranch, None)
-                        if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev:
+                            if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev:
-                            continue
+                                continue
 
-                        urldir = layer.get_fetch_dir()
+                        logger.info('Updating layer dependencies for %s on branch %s' % (layer.name, branch))
-                        repodir = os.path.join(fetchdir, urldir)
+                        cmd = prepare_update_layer_command(options, branchobj, layer, updatedeps=True)
-
+                        logger.debug('Running update dependencies command: %s' % cmd)
-                        utils.checkout_layer_branch(layerbranch, repodir, logger)
+                        ret, output = run_command_interruptible(cmd)
-
+                        if ret == 254:
-                        config_data = layerconfparser.parse_layer(layerbranch, repodir)
+                            # Interrupted by user, break out of loop
-                        if not config_data:
+                            break
-                            logger.debug("Layer %s does not appear to have branch %s" % (layer.name, branch))
-                            continue
-
-                        utils.add_dependencies(layerbranch, config_data, logger=logger)
-                        utils.add_recommends(layerbranch, config_data, logger=logger)
-                finally:
-                    layerconfparser.shutdown()
 
         finally:
             utils.unlock_file(lockfile)

layerindex/update_layer.py

@@ -20,6 +20,7 @@ from distutils.version import LooseVersion
 import itertools
 import utils
 import recipeparse
+import layerconfparse
 
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
@@ -170,6 +171,9 @@ def main():
     parser.add_option("-n", "--dry-run",
             help = "Don't write any data back to the database",
             action="store_true", dest="dryrun")
+    parser.add_option("", "--update-dependencies",
+            help = "Update layer dependencies only",
+            action="store_true", dest="updatedeps")
     parser.add_option("", "--nocheckout",
             help = "Don't check out branches",
             action="store_true", dest="nocheckout")
@@ -238,6 +242,30 @@ def main():
                     branchname = layerbranch.actual_branch
                     branchdesc = "%s (%s)" % (options.branch, branchname)
 
+            if options.updatedeps:
+                # Update layer dependencies only
+                if not layerbranch:
+                    logger.debug('Skipping dependency update for layer %s on branch %s - no layerbranch record' % (layer, branchdesc))
+                    sys.exit(0)
+                if not options.nocheckout:
+                    utils.checkout_layer_branch(layerbranch, repodir, logger=logger)
+                layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
+                if not os.path.exists(layerdir):
+                    # If this happens it was already flagged during the main update, so ignore it
+                    logger.debug('Skipping dependency update for layer %s on branch %s - layer directory not found' % (layer, branchdesc))
+                    sys.exit(0)
+
+                layerconfparser = layerconfparse.LayerConfParse(logger=logger, bitbakepath=bitbakepath, tinfoil=tinfoil)
+                config_data = layerconfparser.parse_layer(layerbranch, layerdir)
+                if not config_data:
+                    logger.debug("Layer %s does not appear to be valid for branch %s" % (layer.name, branchdesc))
+                    sys.exit(0)
+
+                utils.add_dependencies(layerbranch, config_data, logger=logger)
+                utils.add_recommends(layerbranch, config_data, logger=logger)
+
+                sys.exit(0)
+
             # Collect repo info
             repo = git.Repo(repodir)
             assert repo.bare == False
@@ -300,8 +328,7 @@ def main():
             if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
                 # Check out appropriate branch
                 if not options.nocheckout:
-                    out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
+                    utils.checkout_layer_branch(layerbranch, repodir, logger=logger)
-                    out = utils.runcmd("git clean -f -x", repodir, logger=logger)
 
                 if layerbranch.vcs_subdir and not os.path.exists(layerdir):
                     if newbranch:
@@ -316,8 +343,7 @@ def main():
 
                 logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
 
-                from layerconfparse import LayerConfParse
+                layerconfparser = layerconfparse.LayerConfParse(logger=logger, tinfoil=tinfoil)
-                layerconfparser = LayerConfParse(logger=logger, tinfoil=tinfoil)
                 layer_config_data = layerconfparser.parse_layer(layerbranch, layerdir)
                 if not layer_config_data:
                     logger.info("Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc))