RRS: enable grouping recipe upgrades by license

Going back in OE-Core recipe upgrade history, we kept GPLv2 and GPLv3
versions of a number of recipes around, so this is the source of quite a
few situations where we had multiple versions of recipes with the same
recipe name around. Add means of grouping upgrades by license so that we
can keep these versions separate in the upgrade history instead of
detecting lots of apparent upgrades and downgrades if they are
intermingled.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

rrs/migrations/0029_rrs_license_group.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-10-31 03:49
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rrs', '0028_recipeupgrade_srcrev'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='recipeupgrade',
+            name='license',
+            field=models.CharField(blank=True, max_length=100),
+        ),
+        migrations.AddField(
+            model_name='recipeupgradegrouprule',
+            name='license',
+            field=models.CharField(blank=True, help_text='Regular expression to split license on', max_length=100),
+        ),
+        migrations.AddField(
+            model_name='recipeupgradegrouprule',
+            name='priority',
+            field=models.IntegerField(blank=True, help_text='Order to apply rule in (higher first)', null=True),
+        ),
+        migrations.AlterField(
+            model_name='recipeupgradegrouprule',
+            name='pn',
+            field=models.CharField(blank=True, help_text='Regular expression to match recipe to apply to', max_length=100),
+        ),
+        migrations.AlterField(
+            model_name='recipeupgradegrouprule',
+            name='version',
+            field=models.CharField(blank=True, help_text='Regular expression to split version component on', max_length=100),
+        ),
+        migrations.AlterModelOptions(
+            name='recipeupgradegrouprule',
+            options={'ordering': ['-priority']},
+        ),
+    ]

rrs/models.py

@@ -439,22 +439,44 @@ class RecipeUpgradeGroup(models.Model):
 
 class RecipeUpgradeGroupRule(models.Model):
     layerbranch = models.ForeignKey(LayerBranch)
-    pn = models.CharField(max_length=100, help_text='Regular expression to match recipe to apply to')
+    pn = models.CharField(max_length=100, blank=True, help_text='Regular expression to match recipe to apply to')
-    version = models.CharField(max_length=100, help_text='Regular expression to split version component on')
+    version = models.CharField(max_length=100, blank=True, help_text='Regular expression to split version component on')
+    license = models.CharField(max_length=100, blank=True, help_text='Regular expression to split license on')
+    priority = models.IntegerField(blank=True, null=True, help_text='Order to apply rule in (higher first)')
+
+    class Meta:
+        ordering = ["-priority"]
 
     @staticmethod
-    def group_for_params(recipesymbol, version):
+    def group_for_params(recipesymbol, version, license):
-        for rule in RecipeUpgradeGroupRule.objects.filter(layerbranch=recipesymbol.layerbranch):
+        for rule in RecipeUpgradeGroupRule.objects.filter(layerbranch=recipesymbol.layerbranch).order_by('-priority'):
-            if re.match(rule.pn, recipesymbol.pn):
+            pnmatch = None
-                res = re.match(rule.version, version)
+            if rule.pn:
-                if res:
+                pnmatch = re.match(rule.pn, recipesymbol.pn)
-                    if res.groups():
+            if (not rule.pn) or pnmatch:
-                        match = res.groups()[0]
+                if rule.version:
-                    else:
+                    res = re.match(rule.version, version)
-                        match = res.string[res.start(0):res.end(0)]
+                    if res:
-                    group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match)
+                        if res.groups():
-                    group.save()
+                            match = res.groups()[0]
-                    return group
+                        else:
+                            match = res.string[res.start(0):res.end(0)]
+                        group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match)
+                        group.save()
+                        return group
+                elif rule.license:
+                    res = re.match(rule.license, license)
+                    if res:
+                        if res.groups():
+                            match = res.groups()[0]
+                        else:
+                            match = res.string[res.start(0):res.end(0)]
+                        group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match)
+                        group.save()
+                        return group
+                elif pnmatch:
+                    # Allow for dummy rules
+                    return None
         return None
 
     def __str__(self):
@@ -476,6 +498,7 @@ class RecipeUpgrade(models.Model):
     title = models.CharField(max_length=1024, blank=True)
     version = models.CharField(max_length=100, blank=True)
     srcrev = models.CharField(max_length=64, blank=True)
+    license = models.CharField(max_length=100, blank=True)
     author_date = models.DateTimeField(db_index=True)
     commit_date = models.DateTimeField(db_index=True)
     upgrade_type = models.CharField(max_length=1, choices=UPGRADE_TYPE_CHOICES, default='U', db_index=True)
@@ -498,7 +521,7 @@ class RecipeUpgrade(models.Model):
         return self.recipesymbol.layerbranch.commit_url(self.sha1)
 
     def regroup(self):
-        group = RecipeUpgradeGroupRule.group_for_params(self.recipesymbol, self.version)
+        group = RecipeUpgradeGroupRule.group_for_params(self.recipesymbol, self.version, self.license)
         if group != self.group:
             self.group = group
             return True

rrs/tools/dump_upgrades.py

@@ -74,7 +74,7 @@ def main():
         details = []
         for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('group', '-commit_date', '-id'):
             details.append(rrs.views._get_recipe_upgrade_detail(maintplan, ru))
-        details.sort(key=lambda s: list(map(int, s.group.title.split('.') if s.group else [])), reverse=True)
+        details.sort(key=lambda s: rrs.views.RecipeUpgradeGroupSortItem(s.group), reverse=True)
         group = None
         for rud in details:
             if rud.group != group:

rrs/tools/upgrade_history_internal.py

@@ -180,7 +180,7 @@ oecore_bad_revs_2 = [
 """
     Store upgrade into RecipeUpgrade model.
 """
-def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None):
+def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None):
     from rrs.models import Maintainer, RecipeUpgrade
 
     maintainer_name = info.split(';')[0]
@@ -206,6 +206,7 @@ def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, commit, title, info, fi
         upgrade.orig_filepath = orig_filepath
     if prev_version:
         upgrade.prev_version = prev_version
+    upgrade.license = license
     upgrade.regroup()
     upgrade.save()
 
@@ -221,6 +222,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
     srcrev = recipe_data.getVar('SRCREV', True)
     if srcrev == 'INVALID':
         srcrev = ''
+    license = recipe_data.getVar('LICENSE', True)
 
     if '..' in pv or pv.endswith('.'):
         logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct))
@@ -231,10 +233,10 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
 
     all_rupgrades = RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(sha1=ct)
     rupgrades = all_rupgrades
-    group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv)
+    group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv, license)
     if group:
         rupgrades = all_rupgrades.filter(group=group)
-    latest_upgrade = rupgrades.order_by('-commit_date').first()
+    latest_upgrade = rupgrades.order_by('-commit_date', '-id').first()
     if latest_upgrade:
         prev_pv = latest_upgrade.version
         prev_srcrev = latest_upgrade.srcrev
@@ -244,7 +246,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
 
     if prev_pv is None:
         logger.debug("%s: Initial upgrade ( -> %s)." % (pn, pv))
-        _save_upgrade(recipesymbol, layerbranch, pv, srcrev, ct, title, info, filepath, logger)
+        _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger)
     else:
         from common import get_recipe_pv_without_srcpv
 
@@ -269,7 +271,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
                     latest_upgrade.save()
                 else:
                     # Check if the "new" version is already in the database
-                    same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date').last()
+                    same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date', '-id').first()
                     if same_pv_upgrade and \
                             not all_rupgrades.filter(prev_version=pv, commit_date__gt=same_pv_upgrade.commit_date).exists() \
                             and \
@@ -283,7 +285,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
                     op = {'U': 'upgrade', 'D': 'downgrade'}[upgrade_type]
                     logger.debug("%s: detected %s (%s -> %s)" \
                             " in ct %s." % (pn, op, prev_pv, pv, ct))
-                    _save_upgrade(recipesymbol, layerbranch, pv, srcrev, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv)
+                    _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv)
         except KeyboardInterrupt:
             raise
         except Exception as e:
@@ -481,7 +483,7 @@ def generate_history(options, layerbranch_id, commit, logger):
                 # Handle recipes where PN has changed
                 for a, b in moved:
                     logger.debug('Move %s -> %s' % (a,b))
-                    rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date')
+                    rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
                     recipe_data = fn_data.get(b, None)
                     if recipe_data:
                         pn = recipe_data.getVar('PN', True)
@@ -499,13 +501,13 @@ def generate_history(options, layerbranch_id, commit, logger):
                     pn = recipe_data.getVar('PN', True)
                     filepath = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
                     # Check if PN has changed internally
-                    rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date')
+                    rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date', '-id')
                     deleted_pns = rus.filter(upgrade_type__in=['R', 'N']).values_list('recipesymbol__pn', flat=True).distinct()
                     for ru in rus:
                         if ru.recipesymbol.pn != pn and ru.recipesymbol.pn not in deleted_pns and ru.upgrade_type not in ['R', 'N']:
                             # PN changed (set within recipe), we need to mark the old recipe as deleted
                             logger.debug('PN changed (without move): %s -> %s' % (ru.recipesymbol.pn, pn))
-                            _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, ru.filepath, logger, upgrade_type='R')
+                            _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, ru.filepath, logger, upgrade_type='R')
                     orig_filepath = None
                     for a, b in moved:
                         if b == filepath:
@@ -518,19 +520,19 @@ def generate_history(options, layerbranch_id, commit, logger):
                 # Handle recipes that have been moved without it being an upgrade/delete
                 for a, b in moved:
                     if a not in deleted:
-                        rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date')
+                        rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
                         if rus:
                             ru = rus.first()
                             if not RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, filepath=b).exists():
                                 # Need to record the move, otherwise we won't be able to
                                 # find the record if we need to mark the recipe as deleted later
-                                _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
+                                _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
 
                 # Handle deleted recipes
                 for df in deleted:
-                    rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date')
+                    rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date', '-id')
                     for ru in rus:
-                        other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gt=ru.commit_date).exclude(filepath=df).order_by('-commit_date')
+                        other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gt=ru.commit_date).exclude(filepath=df).order_by('-commit_date', '-id')
                         # We make a distinction between deleting just one version and the entire recipe being deleted
                         upgrade_type = 'R'
                         for other_ru in other_rus:
@@ -544,7 +546,7 @@ def generate_history(options, layerbranch_id, commit, logger):
                             continue
                         if ru.upgrade_type != upgrade_type and ru.recipesymbol.pn not in seen_pns:
                             logger.debug("%s: marking as deleted (%s)" % (ru.recipesymbol.pn, ru.filepath))
-                            _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, df, logger, upgrade_type=upgrade_type)
+                            _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, df, logger, upgrade_type=upgrade_type)
                             break
 
                 if options.dry_run:

rrs/views.py

@@ -612,6 +612,36 @@ def recipes_report(request, maintplan_name, release_name, milestone_name):
 
     return response
 
+
+
+class RecipeUpgradeGroupSortItem:
+    def __init__(self, group):
+        self.group = group
+        self.ver = RecipeUpgradeGroupSortItem.group_to_ver(self.group)
+
+    @staticmethod
+    def group_to_ver(grp):
+        if not grp:
+            return []
+        else:
+            try:
+                return list(map(int, grp.title.split('.')))
+            except ValueError:
+                return grp.recipeupgrade_set.exclude(upgrade_type__in=['N','R']).order_by('-id').values_list('id', flat=True).first()
+
+    def __lt__(self, other):
+        if type(self.ver) == type(other.ver):
+            return self.ver < other.ver
+        elif isinstance(self.ver, str) and isinstance(other.ver, list):
+            return True
+        elif isinstance(self.ver, int) and isinstance(other.ver, int):
+            return False
+
+    def __repr__(self):
+        return repr(self.group)
+
+
+
 class RecipeUpgradeDetail():
     title = None
     version = None
@@ -735,9 +765,17 @@ class RecipeDetailView(DetailView):
             context['maintainer_name'] = 'No maintainer'
 
         details = []
+        multigroup = False
+        lastgroup = '' # can't use None here
         for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('group', '-commit_date', '-id'):
             details.append(_get_recipe_upgrade_detail(maintplan, ru))
-        details.sort(key=lambda s: list(map(int, s.group.title.split('.') if s.group else [])), reverse=True)
+            if not multigroup:
+                if lastgroup == '':
+                    lastgroup = ru.group
+                elif ru.group != lastgroup:
+                    multigroup = True
+        details.sort(key=lambda s: RecipeUpgradeGroupSortItem(s.group), reverse=True)
+        context['multigroup'] = multigroup
         context['recipe_upgrade_details'] = details
         context['recipe_upgrade_detail_count'] = len(details)
 

templates/rrs/recipedetail.html

@@ -93,9 +93,11 @@
                     <th>Commit</th>
                 </tr>
                 {% for rud in recipe_upgrade_details %}
+                    {% if multigroup %}
                     {% ifchanged rud.group %}
                 <tr><td colspan="6"><b>{{ rud.group.title }}</b></td></tr>
                     {% endifchanged %}
+                    {% endif %}
                 <tr>
                     <td>{{ rud.title }}</td>
                     <td>{% if rud.upgrade_type != 'R' %}{{ rud.version }}{% if rud.upgrade_type == 'D' %} <span class="label label-warning">downgrade</span>{% endif %}{% endif %}</td>