From 254dc0c7db5a57aedc4b55aae74c0a4ea06ce89b Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Thu, 31 Oct 2019 16:30:52 +1300 Subject: [PATCH] RRS: enable grouping recipe upgrades by license Going back in OE-Core recipe upgrade history, we kept GPLv2 and GPLv3 versions of a number of recipes around, so this is the source of quite a few situations where we had multiple versions of recipes with the same recipe name around. Add means of grouping upgrades by license so that we can keep these versions separate in the upgrade history instead of detecting lots of apparent upgrades and downgrades if they are intermingled. Signed-off-by: Paul Eggleton --- rrs/migrations/0029_rrs_license_group.py | 44 ++++++++++++++++++++ rrs/models.py | 53 +++++++++++++++++------- rrs/tools/dump_upgrades.py | 2 +- rrs/tools/upgrade_history_internal.py | 30 +++++++------- rrs/views.py | 40 +++++++++++++++++- templates/rrs/recipedetail.html | 2 + 6 files changed, 140 insertions(+), 31 deletions(-) create mode 100644 rrs/migrations/0029_rrs_license_group.py diff --git a/rrs/migrations/0029_rrs_license_group.py b/rrs/migrations/0029_rrs_license_group.py new file mode 100644 index 0000000..6ae8c98 --- /dev/null +++ b/rrs/migrations/0029_rrs_license_group.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-10-31 03:49 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rrs', '0028_recipeupgrade_srcrev'), + ] + + operations = [ + migrations.AddField( + model_name='recipeupgrade', + name='license', + field=models.CharField(blank=True, max_length=100), + ), + migrations.AddField( + model_name='recipeupgradegrouprule', + name='license', + field=models.CharField(blank=True, help_text='Regular expression to split license on', max_length=100), + ), + migrations.AddField( + model_name='recipeupgradegrouprule', + name='priority', + field=models.IntegerField(blank=True, help_text='Order to apply rule in (higher first)', null=True), + ), + migrations.AlterField( + model_name='recipeupgradegrouprule', + name='pn', + field=models.CharField(blank=True, help_text='Regular expression to match recipe to apply to', max_length=100), + ), + migrations.AlterField( + model_name='recipeupgradegrouprule', + name='version', + field=models.CharField(blank=True, help_text='Regular expression to split version component on', max_length=100), + ), + migrations.AlterModelOptions( + name='recipeupgradegrouprule', + options={'ordering': ['-priority']}, + ), + ] diff --git a/rrs/models.py b/rrs/models.py index 6033c21..ba7e63c 100644 --- a/rrs/models.py +++ b/rrs/models.py @@ -439,22 +439,44 @@ class RecipeUpgradeGroup(models.Model): class RecipeUpgradeGroupRule(models.Model): layerbranch = models.ForeignKey(LayerBranch) - pn = models.CharField(max_length=100, help_text='Regular expression to match recipe to apply to') - version = models.CharField(max_length=100, help_text='Regular expression to split version component on') + pn = models.CharField(max_length=100, blank=True, help_text='Regular expression to match recipe to apply to') + version = models.CharField(max_length=100, blank=True, help_text='Regular expression to split version component on') + license = models.CharField(max_length=100, blank=True, help_text='Regular expression to split license on') + priority = models.IntegerField(blank=True, null=True, help_text='Order to apply rule in (higher first)') + + class Meta: + ordering = ["-priority"] @staticmethod - def group_for_params(recipesymbol, version): - for rule in RecipeUpgradeGroupRule.objects.filter(layerbranch=recipesymbol.layerbranch): - if re.match(rule.pn, recipesymbol.pn): - res = re.match(rule.version, version) - if res: - if res.groups(): - match = res.groups()[0] - else: - match = res.string[res.start(0):res.end(0)] - group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match) - group.save() - return group + def group_for_params(recipesymbol, version, license): + for rule in RecipeUpgradeGroupRule.objects.filter(layerbranch=recipesymbol.layerbranch).order_by('-priority'): + pnmatch = None + if rule.pn: + pnmatch = re.match(rule.pn, recipesymbol.pn) + if (not rule.pn) or pnmatch: + if rule.version: + res = re.match(rule.version, version) + if res: + if res.groups(): + match = res.groups()[0] + else: + match = res.string[res.start(0):res.end(0)] + group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match) + group.save() + return group + elif rule.license: + res = re.match(rule.license, license) + if res: + if res.groups(): + match = res.groups()[0] + else: + match = res.string[res.start(0):res.end(0)] + group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match) + group.save() + return group + elif pnmatch: + # Allow for dummy rules + return None return None def __str__(self): @@ -476,6 +498,7 @@ class RecipeUpgrade(models.Model): title = models.CharField(max_length=1024, blank=True) version = models.CharField(max_length=100, blank=True) srcrev = models.CharField(max_length=64, blank=True) + license = models.CharField(max_length=100, blank=True) author_date = models.DateTimeField(db_index=True) commit_date = models.DateTimeField(db_index=True) upgrade_type = models.CharField(max_length=1, choices=UPGRADE_TYPE_CHOICES, default='U', db_index=True) @@ -498,7 +521,7 @@ class RecipeUpgrade(models.Model): return self.recipesymbol.layerbranch.commit_url(self.sha1) def regroup(self): - group = RecipeUpgradeGroupRule.group_for_params(self.recipesymbol, self.version) + group = RecipeUpgradeGroupRule.group_for_params(self.recipesymbol, self.version, self.license) if group != self.group: self.group = group return True diff --git a/rrs/tools/dump_upgrades.py b/rrs/tools/dump_upgrades.py index f40f46c..011b91e 100755 --- a/rrs/tools/dump_upgrades.py +++ b/rrs/tools/dump_upgrades.py @@ -74,7 +74,7 @@ def main(): details = [] for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('group', '-commit_date', '-id'): details.append(rrs.views._get_recipe_upgrade_detail(maintplan, ru)) - details.sort(key=lambda s: list(map(int, s.group.title.split('.') if s.group else [])), reverse=True) + details.sort(key=lambda s: rrs.views.RecipeUpgradeGroupSortItem(s.group), reverse=True) group = None for rud in details: if rud.group != group: diff --git a/rrs/tools/upgrade_history_internal.py b/rrs/tools/upgrade_history_internal.py index 9129520..afd0b1c 100644 --- a/rrs/tools/upgrade_history_internal.py +++ b/rrs/tools/upgrade_history_internal.py @@ -180,7 +180,7 @@ oecore_bad_revs_2 = [ """ Store upgrade into RecipeUpgrade model. """ -def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None): +def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None): from rrs.models import Maintainer, RecipeUpgrade maintainer_name = info.split(';')[0] @@ -206,6 +206,7 @@ def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, commit, title, info, fi upgrade.orig_filepath = orig_filepath if prev_version: upgrade.prev_version = prev_version + upgrade.license = license upgrade.regroup() upgrade.save() @@ -221,6 +222,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, srcrev = recipe_data.getVar('SRCREV', True) if srcrev == 'INVALID': srcrev = '' + license = recipe_data.getVar('LICENSE', True) if '..' in pv or pv.endswith('.'): logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct)) @@ -231,10 +233,10 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, all_rupgrades = RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(sha1=ct) rupgrades = all_rupgrades - group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv) + group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv, license) if group: rupgrades = all_rupgrades.filter(group=group) - latest_upgrade = rupgrades.order_by('-commit_date').first() + latest_upgrade = rupgrades.order_by('-commit_date', '-id').first() if latest_upgrade: prev_pv = latest_upgrade.version prev_srcrev = latest_upgrade.srcrev @@ -244,7 +246,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, if prev_pv is None: logger.debug("%s: Initial upgrade ( -> %s)." % (pn, pv)) - _save_upgrade(recipesymbol, layerbranch, pv, srcrev, ct, title, info, filepath, logger) + _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger) else: from common import get_recipe_pv_without_srcpv @@ -269,7 +271,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, latest_upgrade.save() else: # Check if the "new" version is already in the database - same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date').last() + same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date', '-id').first() if same_pv_upgrade and \ not all_rupgrades.filter(prev_version=pv, commit_date__gt=same_pv_upgrade.commit_date).exists() \ and \ @@ -283,7 +285,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, op = {'U': 'upgrade', 'D': 'downgrade'}[upgrade_type] logger.debug("%s: detected %s (%s -> %s)" \ " in ct %s." % (pn, op, prev_pv, pv, ct)) - _save_upgrade(recipesymbol, layerbranch, pv, srcrev, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv) + _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv) except KeyboardInterrupt: raise except Exception as e: @@ -481,7 +483,7 @@ def generate_history(options, layerbranch_id, commit, logger): # Handle recipes where PN has changed for a, b in moved: logger.debug('Move %s -> %s' % (a,b)) - rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date') + rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id') recipe_data = fn_data.get(b, None) if recipe_data: pn = recipe_data.getVar('PN', True) @@ -499,13 +501,13 @@ def generate_history(options, layerbranch_id, commit, logger): pn = recipe_data.getVar('PN', True) filepath = os.path.relpath(recipe_data.getVar('FILE', True), repodir) # Check if PN has changed internally - rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date') + rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date', '-id') deleted_pns = rus.filter(upgrade_type__in=['R', 'N']).values_list('recipesymbol__pn', flat=True).distinct() for ru in rus: if ru.recipesymbol.pn != pn and ru.recipesymbol.pn not in deleted_pns and ru.upgrade_type not in ['R', 'N']: # PN changed (set within recipe), we need to mark the old recipe as deleted logger.debug('PN changed (without move): %s -> %s' % (ru.recipesymbol.pn, pn)) - _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, ru.filepath, logger, upgrade_type='R') + _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, ru.filepath, logger, upgrade_type='R') orig_filepath = None for a, b in moved: if b == filepath: @@ -518,19 +520,19 @@ def generate_history(options, layerbranch_id, commit, logger): # Handle recipes that have been moved without it being an upgrade/delete for a, b in moved: if a not in deleted: - rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date') + rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id') if rus: ru = rus.first() if not RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, filepath=b).exists(): # Need to record the move, otherwise we won't be able to # find the record if we need to mark the recipe as deleted later - _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a) + _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a) # Handle deleted recipes for df in deleted: - rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date') + rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date', '-id') for ru in rus: - other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gt=ru.commit_date).exclude(filepath=df).order_by('-commit_date') + other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gt=ru.commit_date).exclude(filepath=df).order_by('-commit_date', '-id') # We make a distinction between deleting just one version and the entire recipe being deleted upgrade_type = 'R' for other_ru in other_rus: @@ -544,7 +546,7 @@ def generate_history(options, layerbranch_id, commit, logger): continue if ru.upgrade_type != upgrade_type and ru.recipesymbol.pn not in seen_pns: logger.debug("%s: marking as deleted (%s)" % (ru.recipesymbol.pn, ru.filepath)) - _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, df, logger, upgrade_type=upgrade_type) + _save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, df, logger, upgrade_type=upgrade_type) break if options.dry_run: diff --git a/rrs/views.py b/rrs/views.py index 922ca54..9c36a48 100644 --- a/rrs/views.py +++ b/rrs/views.py @@ -612,6 +612,36 @@ def recipes_report(request, maintplan_name, release_name, milestone_name): return response + + +class RecipeUpgradeGroupSortItem: + def __init__(self, group): + self.group = group + self.ver = RecipeUpgradeGroupSortItem.group_to_ver(self.group) + + @staticmethod + def group_to_ver(grp): + if not grp: + return [] + else: + try: + return list(map(int, grp.title.split('.'))) + except ValueError: + return grp.recipeupgrade_set.exclude(upgrade_type__in=['N','R']).order_by('-id').values_list('id', flat=True).first() + + def __lt__(self, other): + if type(self.ver) == type(other.ver): + return self.ver < other.ver + elif isinstance(self.ver, str) and isinstance(other.ver, list): + return True + elif isinstance(self.ver, int) and isinstance(other.ver, int): + return False + + def __repr__(self): + return repr(self.group) + + + class RecipeUpgradeDetail(): title = None version = None @@ -735,9 +765,17 @@ class RecipeDetailView(DetailView): context['maintainer_name'] = 'No maintainer' details = [] + multigroup = False + lastgroup = '' # can't use None here for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('group', '-commit_date', '-id'): details.append(_get_recipe_upgrade_detail(maintplan, ru)) - details.sort(key=lambda s: list(map(int, s.group.title.split('.') if s.group else [])), reverse=True) + if not multigroup: + if lastgroup == '': + lastgroup = ru.group + elif ru.group != lastgroup: + multigroup = True + details.sort(key=lambda s: RecipeUpgradeGroupSortItem(s.group), reverse=True) + context['multigroup'] = multigroup context['recipe_upgrade_details'] = details context['recipe_upgrade_detail_count'] = len(details) diff --git a/templates/rrs/recipedetail.html b/templates/rrs/recipedetail.html index 2c15553..e2bcfe2 100644 --- a/templates/rrs/recipedetail.html +++ b/templates/rrs/recipedetail.html @@ -93,9 +93,11 @@ Commit {% for rud in recipe_upgrade_details %} + {% if multigroup %} {% ifchanged rud.group %} {{ rud.group.title }} {% endifchanged %} + {% endif %} {{ rud.title }} {% if rud.upgrade_type != 'R' %}{{ rud.version }}{% if rud.upgrade_type == 'D' %} downgrade{% endif %}{% endif %}