update: fix handling of moves outside of a layer

If a file is moved (renamed) to a path outside of the layer, e.g.
another layer within a multi-layer repository, then we need to treat it
as a delete. Up until now we were updating the path and continuing, and
then the recipe was also picked up as an add in the other layer, leading
to duplicate recipe entries. I'd noticed these duplicates before but up
until now I'd thought that they were due to another bug we already
fixed, apparently not.

In order to remove these erroneous duplicate entries in existing
databases I have also added a layerindex/tools/fixup_duplicates.py
script. I've also made the -r/--reload option delete them as well.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

layerindex/tools/fixup_duplicates.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+# Fix recipes that were moved out
+#
+# Copyright (C) 2017 Intel Corporation
+# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
+#
+# Licensed under the MIT license, see COPYING.MIT for details
+
+
+import sys
+import os
+
+sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
+
+import optparse
+import utils
+import logging
+
+class DryRunRollbackException(Exception):
+    pass
+
+logger = utils.logger_create('LayerIndexFixup')
+
+
+
+def main():
+    parser = optparse.OptionParser(
+        usage = """
+    %prog [options""")
+
+    parser.add_option("-n", "--dry-run",
+            help = "Don't write any data back to the database",
+            action="store_true", dest="dryrun")
+    parser.add_option("-d", "--debug",
+            help = "Enable debug output",
+            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
+    parser.add_option("-q", "--quiet",
+            help = "Hide all output except error messages",
+            action="store_const", const=logging.ERROR, dest="loglevel")
+
+    options, args = parser.parse_args(sys.argv)
+
+    utils.setup_django()
+    import settings
+    from layerindex.models import Recipe
+    from django.db import transaction
+
+    logger.setLevel(options.loglevel)
+
+    try:
+        with transaction.atomic():
+            #LayerBranch.objects.filter(layermaintainer__isnull=True).delete()
+            #LayerItem.objects.filter(layerbranch__isnull=True).filter(classic=False).delete()
+            #LayerItem.objects.filter(layerbranch__isnull=True).filter(classic=False).delete()
+            for recipe in Recipe.objects.filter(filepath__startswith='../'):
+                print('Deleting erroneous recipe %s %s' % (recipe.layerbranch, recipe))
+                recipe.delete()
+
+            if options.dryrun:
+                raise DryRunRollbackException()
+    except DryRunRollbackException:
+        pass
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

layerindex/update_layer.py

@@ -502,6 +502,10 @@ def main():
                         if skip:
                             continue
                         if oldpath.startswith(subdir_start):
+                            if not newpath.startswith(subdir_start):
+                                logger.debug("Treating rename of %s to %s as a delete since new path is outside layer" % (oldpath, newpath))
+                                other_deletes.append(diffitem)
+                                continue
                             (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
                             (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
                             if oldtypename != newtypename:
@@ -684,16 +688,21 @@ def main():
                         # First, check which recipes still exist
                         layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
                         for v in layerrecipe_values:
-                            root = os.path.join(layerdir, v['filepath'])
-                            fullpath = os.path.join(root, v['filename'])
-                            preserve = True
-                            if os.path.exists(fullpath):
-                                for removedir in removedirs:
-                                    if fullpath.startswith(removedir):
-                                        preserve = False
-                                        break
-                            else:
+                            if v['filepath'].startswith('../'):
+                                # FIXME: These recipes were present due to a bug (not handling renames
+                                # to paths outside the layer) - this can be removed at some point in the future
                                 preserve = False
+                            else:
+                                root = os.path.join(layerdir, v['filepath'])
+                                fullpath = os.path.join(root, v['filename'])
+                                if os.path.exists(fullpath):
+                                    preserve = True
+                                    for removedir in removedirs:
+                                        if fullpath.startswith(removedir):
+                                            preserve = False
+                                            break
+                                else:
+                                    preserve = False
 
                             if preserve:
                                 # Recipe still exists, update it