layerindex: improve updates for actual_branch

Some layers now have one branch with many supported
LAYERSERIES_COMPAT. If this branch name does not match
one of the stable releases, LayerBranches might not
have been created. When actual_branch is set, it is
only set in a LayerBranch object. We previously
could not update (create) a stable branch with actual_branch
except manually in the admin interface.

Add --force-create option to be used in conjunction with
--actual-branch (which already requires --branch) in the
update.py script. This tells the script to ignore the
fact that no layerbranch exists already.

Add --actual-branch to update_layer.py so that we can create
(and more importantly checkout) an actual_branch for the
given stable --branch.

Update utils.py to allow checking out of actual_branch when
a LayerBranch does not yet exist.

While we are at it, ensure that any Branch that is marked
as no update will be skipped even with --force-create. The
main reason that a Branch has updates disabled is because the
bitbake or python syntax has changed enough to cause exceptions.

This script can now be run with:

./layerindex/update.py \
  --layer meta-weird-one \
  --branch kirkstone \
  --actual-branch=nonstandard \
  --force-create

Which will attempt to create a meta-weird-one:kirkstone layerbranch
checked out at the 'nonstandard' branch from that layer's git repo.

This allows layerindex admins to at least populate the database
without tedious creation of layerbranches in the admin interface.

Helps make the "branch mapping" actually work and be useful:
[YOCTO #8008]

Signed-off-by: Tim Orling <tim.orling@konsulko.com>

layerindex/update.py

@@ -43,6 +43,8 @@ def prepare_update_layer_command(options, branch, layer, initial=False):
     else:
         cmdprefix = 'python3'
     cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name)
+    if options.actual_branch and options.force_create:
+        cmd += ' --actual-branch=%s' % options.actual_branch
     if options.reload:
         cmd += ' --reload'
     if options.fullreload:
@@ -86,15 +88,18 @@ def update_actual_branch(layerquery, fetchdir, branch, options, update_bitbake,
             logger.info("Skipping update actual_branch for %s - branch %s doesn't exist" % (layer.name, actual_branch))
             continue
         layerbranch = layer.get_layerbranch(branch)
-        if not layerbranch:
-            logger.info("Skipping update actual_branch for %s - layerbranch %s doesn't exist" % (layer.name, branch))
-            continue
-        if actual_branch != layerbranch.actual_branch:
-            logger.info("%s: %s.actual_branch: %s -> %s" % (layer.name, branch, layerbranch.actual_branch, actual_branch))
-            layerbranch.actual_branch = actual_branch
-            to_save.add(layerbranch)
+        if not options.force_create:
+            if not layerbranch:
+                logger.info("Skipping update actual_branch for %s - layerbranch %s doesn't exist" % (layer.name, branch))
+                continue
+            if actual_branch != layerbranch.actual_branch:
+                logger.info("%s: %s.actual_branch: %s -> %s" % (layer.name, branch, layerbranch.actual_branch, actual_branch))
+                layerbranch.actual_branch = actual_branch
+                to_save.add(layerbranch)
+            else:
+                logger.info("%s: %s.actual_branch is already %s, so no change" % (layer.name, branch, actual_branch))
         else:
-            logger.info("%s: %s.actual_branch is already %s, so no change" % (layer.name, branch, actual_branch))
+            logger.info("%s: Allowing branch %s with actual_branch %s to attempt to be created" % (layer.name, branch, actual_branch))
 
     # At last, do the save
     if not options.dryrun:
@@ -169,6 +174,9 @@ def main():
     parser.add_option("-a", "--actual-branch",
             help = "Update actual branch for layer and bitbake",
             action="store", dest="actual_branch", default='')
+    parser.add_option("", "--force-create",
+            help = "Create layer branch if it does not already exist",
+            action="store_true", dest="force_create", default=False)
     parser.add_option("-d", "--debug",
             help = "Enable debug output",
             action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
@@ -197,6 +205,10 @@ def main():
             if not utils.get_branch(branch):
                 logger.error("Specified branch %s is not valid" % branch)
                 sys.exit(1)
+            branchquery = Branch.objects.filter(updates_enabled=True).filter(name=branch)
+            if not branchquery.count() > 0:
+                logger.warning("Updates are disabled for specified branch %s" % branch)
+                sys.exit(1)
     else:
         branchquery = Branch.objects.filter(updates_enabled=True)
         branches = [branch.name for branch in branchquery]
@@ -315,7 +327,7 @@ def main():
                     logger.error("No repositories could be fetched, exiting")
                     sys.exit(1)
 
-            if options.actual_branch:
+            if options.actual_branch and not options.force_create:
                 update_actual_branch(layerquery, fetchdir, branches[0], options, update_bitbake, bitbakepath)
                 return
 
@@ -389,6 +401,10 @@ def main():
                         logger.error('Repository %s is bare, not supported' % repodir)
                         continue
                     try:
+                        # Allow stable branches to be created if actual_branch exists
+                        if options.actual_branch:
+                            branchname = options.actual_branch
+                            branchdesc = "%s (%s)" % (branch, branchname)
                         # Always get origin/branchname, so it raises error when branch doesn't exist when nocheckout
                         topcommit = repo.commit('origin/%s' % branchname)
                         if options.nocheckout:
@@ -419,7 +435,10 @@ def main():
                     else:
                         # Check out appropriate branch
                         if not options.nocheckout:
-                            utils.checkout_layer_branch(layerbranch, repodir, logger=logger)
+                            if not options.actual_branch:
+                                utils.checkout_layer_branch(layerbranch, repodir, logger=logger)
+                            else:
+                                utils.checkout_layer_branch(layerbranch, repodir, actual_branch=options.actual_branch, logger=logger)
                         layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
                         if layerbranch.vcs_subdir and not os.path.exists(layerdir):
                             print_subdir_error(newbranch, layer.name, layerbranch.vcs_subdir, branchdesc)

layerindex/update_layer.py

@@ -272,6 +272,9 @@ def main():
     parser.add_option("-i", "--initial",
             help = "Print initial values parsed from layer.conf only",
             action="store_true")
+    parser.add_option("-a", "--actual-branch",
+            help = "Specify actual_branch for git checkout",
+            action="store", dest="actual_branch", default=None)
     parser.add_option("-d", "--debug",
             help = "Enable debug output",
             action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
@@ -326,6 +329,9 @@ def main():
         if layerbranch.actual_branch:
             branchname = layerbranch.actual_branch
             branchdesc = "%s (%s)" % (options.branch, branchname)
+    elif options.actual_branch:
+        branchname = options.actual_branch
+        branchdesc = "%s (%s)" % (options.branch, branchname)
 
     # Collect repo info
     repo = git.Repo(repodir)
@@ -347,6 +353,8 @@ def main():
                 layerbranch = LayerBranch()
                 layerbranch.layer = layer
                 layerbranch.branch = branch
+                if options.actual_branch:
+                    layerbranch.actual_branch = options.actual_branch
                 layerbranch_source = layer.get_layerbranch(branch)
                 if not layerbranch_source:
                     layerbranch_source = layer.get_layerbranch(None)

layerindex/utils.py

@@ -297,8 +297,11 @@ def checkout_repo(repodir, commit, logger, force=False):
         # Now check out the revision
         runcmd(['git', 'checkout', commit], repodir, logger=logger)
 
-def checkout_layer_branch(layerbranch, repodir, logger=None):
-    branchname = layerbranch.get_checkout_branch()
+def checkout_layer_branch(layerbranch, repodir, actual_branch=None, logger=None):
+    if actual_branch:
+        branchname = actual_branch
+    else:
+        branchname = layerbranch.get_checkout_branch()
     checkout_repo(repodir, 'origin/%s' % branchname, logger)
 
 def is_layer_valid(layerdir):