RRS: handle recipe moves without overwriting data

If we ever want to analyse the upgrade chain later on then we need to
avoid overwriting the paths when we identify a moved recipe - instead,
store a "move" upgrade record (not shown in the UI) that we can later
pick up when we are going through and deleting.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

rrs/migrations/0025_recipeupgrade_move.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-08-18 22:59
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rrs', '0024_recipeupgrade_downgrade'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='recipeupgrade',
+            name='orig_filepath',
+            field=models.CharField(blank=True, max_length=512),
+        ),
+        migrations.AlterField(
+            model_name='recipeupgrade',
+            name='upgrade_type',
+            field=models.CharField(choices=[('U', 'Upgrade'), ('D', 'Downgrade'), ('N', 'Delete'), ('R', 'Delete (final)'), ('M', 'Move')], db_index=True, default='U', max_length=1),
+        ),
+    ]

rrs/models.py

@@ -435,6 +435,7 @@ class RecipeUpgrade(models.Model):
         ('D', 'Downgrade'),
         ('N', 'Delete'),
         ('R', 'Delete (final)'),
+        ('M', 'Move'),
     )
 
     recipesymbol = models.ForeignKey(RecipeSymbol)
@@ -446,6 +447,7 @@ class RecipeUpgrade(models.Model):
     commit_date = models.DateTimeField(db_index=True)
     upgrade_type = models.CharField(max_length=1, choices=UPGRADE_TYPE_CHOICES, default='U', db_index=True)
     filepath = models.CharField(max_length=512, blank=True)
+    orig_filepath = models.CharField(max_length=512, blank=True)
 
     @staticmethod
     def get_by_recipe_and_date(recipe, end_date):
@@ -467,6 +469,9 @@ class RecipeUpgrade(models.Model):
         elif self.upgrade_type == 'N':
             return '%s: deleted (%s)' % (self.recipesymbol.pn,
                             self.commit_date)
+        elif self.upgrade_type == 'M':
+            return '%s: moved (%s)' % (self.recipesymbol.pn,
+                            self.commit_date)
         else:
             return '%s: (%s, %s)' % (self.recipesymbol.pn, self.version,
                             self.commit_date)

rrs/tools/upgrade_history_internal.py

@@ -120,7 +120,7 @@ oecore_bad_revs = {
 """
     Store upgrade into RecipeUpgrade model.
 """
-def _save_upgrade(recipesymbol, layerbranch, pv, commit, title, info, filepath, logger, upgrade_type=None):
+def _save_upgrade(recipesymbol, layerbranch, pv, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None):
     from rrs.models import Maintainer, RecipeUpgrade, RecipeSymbol
 
     maintainer_name = info.split(';')[0]
@@ -141,12 +141,14 @@ def _save_upgrade(recipesymbol, layerbranch, pv, commit, title, info, filepath,
     upgrade.filepath = filepath
     if upgrade_type:
         upgrade.upgrade_type = upgrade_type
+    if orig_filepath:
+        upgrade.orig_filepath = orig_filepath
     upgrade.save()
 
 """
     Create upgrade receives new recipe_data and cmp versions.
 """
-def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, pn_recipes, initial=False):
+def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, pn_recipes, initial=False, orig_filepath=None):
     from rrs.models import RecipeUpgrade, RecipeSymbol
     from bb.utils import vercmp_string
 
@@ -214,7 +216,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
                     op = {'U': 'upgrade', 'D': 'downgrade'}[upgrade_type]
                     logger.debug("%s: detected %s (%s -> %s)" \
                             " in ct %s." % (pn, op, prev_pv, pv, ct))
-                    _save_upgrade(recipesymbol, layerbranch, pv, ct, title, info, filepath, logger, upgrade_type=upgrade_type)
+                    _save_upgrade(recipesymbol, layerbranch, pv, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath)
         except KeyboardInterrupt:
             raise
         except Exception as e:
@@ -385,6 +387,7 @@ def generate_history(options, layerbranch_id, commit, logger):
         seen_pns = []
         try:
             with transaction.atomic():
+                # Handle recipes where PN has changed
                 for a, b in moved:
                     logger.debug('Move %s -> %s' % (a,b))
                     rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date')
@@ -400,20 +403,31 @@ def generate_history(options, layerbranch_id, commit, logger):
                     else:
                         logger.warning('Unable to find parsed data for recipe %s' % b)
 
-                    if a not in deleted:
-                        # Need to keep filepath up-to-date, otherwise we won't be able to
-                        # find the record if we need to mark it as deleted later
-                        for ru in rus:
-                            ru.filepath = b
-                            ru.save()
-
+                # Handle recipes that exist at this point in time (which may have upgraded)
                 for recipe_data in recipes:
                     pn = recipe_data.getVar('PN', True)
                     filepath = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
+                    orig_filepath = None
+                    for a, b in moved:
+                        if b == filepath:
+                            orig_filepath = a
+                            break
                     _create_upgrade(recipe_data, layerbranch, recordcommit, title,
-                            info, filepath, logger, pn_data[pn], initial=options.initial)
+                            info, filepath, logger, pn_data[pn], initial=options.initial, orig_filepath=orig_filepath)
                     seen_pns.append(pn)
 
+                # Handle recipes that have been moved without it being an upgrade/delete
+                for a, b in moved:
+                    if a not in deleted:
+                        rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date')
+                        if rus:
+                            ru = rus.first()
+                            if not RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, filepath=b).exists():
+                                # Need to record the move, otherwise we won't be able to
+                                # find the record if we need to mark the recipe as deleted later
+                                _save_upgrade(ru.recipesymbol, layerbranch, ru.version, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
+
+                # Handle deleted recipes
                 for df in deleted:
                     rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date')
                     for ru in rus:

rrs/views.py

@@ -201,7 +201,7 @@ class Raw():
                             ) AS rownum
                         FROM rrs_recipeupgrade
                         WHERE commit_date <= %s
-                        AND upgrade_type <> 'N') AS te
+                        AND upgrade_type not in ('M', 'N')) AS te
                         INNER JOIN rrs_recipesymbol AS rs
                         ON te.recipesymbol_id = rs.id
                         WHERE rownum = 1
@@ -731,7 +731,7 @@ class RecipeDetailView(DetailView):
             context['maintainer_name'] = 'No maintainer'
 
         context['recipe_upgrade_details'] = []
-        for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).order_by('-commit_date'): 
+        for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('-commit_date'):
             context['recipe_upgrade_details'].append(_get_recipe_upgrade_detail(maintplan, ru))
         context['recipe_upgrade_detail_count'] = len(context['recipe_upgrade_details'])