MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-19 20:59:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

RRS: enable grouping recipe upgrades by license

Browse Source 
Going back in OE-Core recipe upgrade history, we kept GPLv2 and GPLv3
versions of a number of recipes around, so this is the source of quite a
few situations where we had multiple versions of recipes with the same
recipe name around. Add means of grouping upgrades by license so that we
can keep these versions separate in the upgrade history instead of
detecting lots of apparent upgrades and downgrades if they are
intermingled.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2019-10-31 16:30:52 +13:00
parent 687099d96f
commit 254dc0c7db
6 changed files with 140 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
rrs/migrations/0029_rrs_license_group.py Normal file
View File

@ -0,0 +1,44 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.22 on 2019-10-31 03:49
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rrs', '0028_recipeupgrade_srcrev'),
]
operations = [
migrations.AddField(
model_name='recipeupgrade',
name='license',
field=models.CharField(blank=True, max_length=100),
),
migrations.AddField(
model_name='recipeupgradegrouprule',
name='license',
field=models.CharField(blank=True, help_text='Regular expression to split license on', max_length=100),
),
migrations.AddField(
model_name='recipeupgradegrouprule',
name='priority',
field=models.IntegerField(blank=True, help_text='Order to apply rule in (higher first)', null=True),
),
migrations.AlterField(
model_name='recipeupgradegrouprule',
name='pn',
field=models.CharField(blank=True, help_text='Regular expression to match recipe to apply to', max_length=100),
),
migrations.AlterField(
model_name='recipeupgradegrouprule',
name='version',
field=models.CharField(blank=True, help_text='Regular expression to split version component on', max_length=100),
),
migrations.AlterModelOptions(
name='recipeupgradegrouprule',
options={'ordering': ['-priority']},
),
]

35
rrs/models.py
View File

@ -439,13 +439,22 @@ class RecipeUpgradeGroup(models.Model):
class RecipeUpgradeGroupRule(models.Model):
layerbranch = models.ForeignKey(LayerBranch)
pn = models.CharField(max_length=100, help_text='Regular expression to match recipe to apply to')
version = models.CharField(max_length=100, help_text='Regular expression to split version component on')
pn = models.CharField(max_length=100, blank=True, help_text='Regular expression to match recipe to apply to')
version = models.CharField(max_length=100, blank=True, help_text='Regular expression to split version component on')
license = models.CharField(max_length=100, blank=True, help_text='Regular expression to split license on')
priority = models.IntegerField(blank=True, null=True, help_text='Order to apply rule in (higher first)')
class Meta:
ordering = ["-priority"]
@staticmethod
def group_for_params(recipesymbol, version):
for rule in RecipeUpgradeGroupRule.objects.filter(layerbranch=recipesymbol.layerbranch):
if re.match(rule.pn, recipesymbol.pn):
def group_for_params(recipesymbol, version, license):
for rule in RecipeUpgradeGroupRule.objects.filter(layerbranch=recipesymbol.layerbranch).order_by('-priority'):
pnmatch = None
if rule.pn:
pnmatch = re.match(rule.pn, recipesymbol.pn)
if (not rule.pn) or pnmatch:
if rule.version:
res = re.match(rule.version, version)
if res:
if res.groups():
@ -455,6 +464,19 @@ class RecipeUpgradeGroupRule(models.Model):
group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match)
group.save()
return group
elif rule.license:
res = re.match(rule.license, license)
if res:
if res.groups():
match = res.groups()[0]
else:
match = res.string[res.start(0):res.end(0)]
group, _ = RecipeUpgradeGroup.objects.get_or_create(recipesymbol=recipesymbol, title=match)
group.save()
return group
elif pnmatch:
# Allow for dummy rules
return None
return None
def __str__(self):
@ -476,6 +498,7 @@ class RecipeUpgrade(models.Model):
title = models.CharField(max_length=1024, blank=True)
version = models.CharField(max_length=100, blank=True)
srcrev = models.CharField(max_length=64, blank=True)
license = models.CharField(max_length=100, blank=True)
author_date = models.DateTimeField(db_index=True)
commit_date = models.DateTimeField(db_index=True)
upgrade_type = models.CharField(max_length=1, choices=UPGRADE_TYPE_CHOICES, default='U', db_index=True)
@ -498,7 +521,7 @@ class RecipeUpgrade(models.Model):
return self.recipesymbol.layerbranch.commit_url(self.sha1)
def regroup(self):
group = RecipeUpgradeGroupRule.group_for_params(self.recipesymbol, self.version)
group = RecipeUpgradeGroupRule.group_for_params(self.recipesymbol, self.version, self.license)
if group != self.group:
self.group = group
return True

2
rrs/tools/dump_upgrades.py
View File

@ -74,7 +74,7 @@ def main():
details = []
for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('group', '-commit_date', '-id'):
details.append(rrs.views._get_recipe_upgrade_detail(maintplan, ru))
details.sort(key=lambda s: list(map(int, s.group.title.split('.') if s.group else [])), reverse=True)
details.sort(key=lambda s: rrs.views.RecipeUpgradeGroupSortItem(s.group), reverse=True)
group = None
for rud in details:
if rud.group != group:

30
rrs/tools/upgrade_history_internal.py
View File

@ -180,7 +180,7 @@ oecore_bad_revs_2 = [
"""
Store upgrade into RecipeUpgrade model.
"""
def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None):
def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None):
from rrs.models import Maintainer, RecipeUpgrade
maintainer_name = info.split(';')[0]
@ -206,6 +206,7 @@ def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, commit, title, info, fi
upgrade.orig_filepath = orig_filepath
if prev_version:
upgrade.prev_version = prev_version
upgrade.license = license
upgrade.regroup()
upgrade.save()
@ -221,6 +222,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
srcrev = recipe_data.getVar('SRCREV', True)
if srcrev == 'INVALID':
srcrev = ''
license = recipe_data.getVar('LICENSE', True)
if '..' in pv or pv.endswith('.'):
logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct))
@ -231,10 +233,10 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
all_rupgrades = RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(sha1=ct)
rupgrades = all_rupgrades
group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv)
group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv, license)
if group:
rupgrades = all_rupgrades.filter(group=group)
latest_upgrade = rupgrades.order_by('-commit_date').first()
latest_upgrade = rupgrades.order_by('-commit_date', '-id').first()
if latest_upgrade:
prev_pv = latest_upgrade.version
prev_srcrev = latest_upgrade.srcrev
@ -244,7 +246,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
if prev_pv is None:
logger.debug("%s: Initial upgrade ( -> %s)." % (pn, pv))
_save_upgrade(recipesymbol, layerbranch, pv, srcrev, ct, title, info, filepath, logger)
_save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger)
else:
from common import get_recipe_pv_without_srcpv
@ -269,7 +271,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
latest_upgrade.save()
else:
# Check if the "new" version is already in the database
same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date').last()
same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date', '-id').first()
if same_pv_upgrade and \
not all_rupgrades.filter(prev_version=pv, commit_date__gt=same_pv_upgrade.commit_date).exists() \
and \
@ -283,7 +285,7 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger,
op = {'U': 'upgrade', 'D': 'downgrade'}[upgrade_type]
logger.debug("%s: detected %s (%s -> %s)" \
" in ct %s." % (pn, op, prev_pv, pv, ct))
_save_upgrade(recipesymbol, layerbranch, pv, srcrev, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv)
_save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv)
except KeyboardInterrupt:
raise
except Exception as e:
@ -481,7 +483,7 @@ def generate_history(options, layerbranch_id, commit, logger):
# Handle recipes where PN has changed
for a, b in moved:
logger.debug('Move %s -> %s' % (a,b))
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date')
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
recipe_data = fn_data.get(b, None)
if recipe_data:
pn = recipe_data.getVar('PN', True)
@ -499,13 +501,13 @@ def generate_history(options, layerbranch_id, commit, logger):
pn = recipe_data.getVar('PN', True)
filepath = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
# Check if PN has changed internally
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date')
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date', '-id')
deleted_pns = rus.filter(upgrade_type__in=['R', 'N']).values_list('recipesymbol__pn', flat=True).distinct()
for ru in rus:
if ru.recipesymbol.pn != pn and ru.recipesymbol.pn not in deleted_pns and ru.upgrade_type not in ['R', 'N']:
# PN changed (set within recipe), we need to mark the old recipe as deleted
logger.debug('PN changed (without move): %s -> %s' % (ru.recipesymbol.pn, pn))
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, ru.filepath, logger, upgrade_type='R')
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, ru.filepath, logger, upgrade_type='R')
orig_filepath = None
for a, b in moved:
if b == filepath:
@ -518,19 +520,19 @@ def generate_history(options, layerbranch_id, commit, logger):
# Handle recipes that have been moved without it being an upgrade/delete
for a, b in moved:
if a not in deleted:
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date')
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
if rus:
ru = rus.first()
if not RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, filepath=b).exists():
# Need to record the move, otherwise we won't be able to
# find the record if we need to mark the recipe as deleted later
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
# Handle deleted recipes
for df in deleted:
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date')
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date', '-id')
for ru in rus:
other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gt=ru.commit_date).exclude(filepath=df).order_by('-commit_date')
other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gt=ru.commit_date).exclude(filepath=df).order_by('-commit_date', '-id')
# We make a distinction between deleting just one version and the entire recipe being deleted
upgrade_type = 'R'
for other_ru in other_rus:
@ -544,7 +546,7 @@ def generate_history(options, layerbranch_id, commit, logger):
continue
if ru.upgrade_type != upgrade_type and ru.recipesymbol.pn not in seen_pns:
logger.debug("%s: marking as deleted (%s)" % (ru.recipesymbol.pn, ru.filepath))
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, recordcommit, title, info, df, logger, upgrade_type=upgrade_type)
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, df, logger, upgrade_type=upgrade_type)
break
if options.dry_run:

40
rrs/views.py
View File

@ -612,6 +612,36 @@ def recipes_report(request, maintplan_name, release_name, milestone_name):
return response
class RecipeUpgradeGroupSortItem:
def __init__(self, group):
self.group = group
self.ver = RecipeUpgradeGroupSortItem.group_to_ver(self.group)
@staticmethod
def group_to_ver(grp):
if not grp:
return []
else:
try:
return list(map(int, grp.title.split('.')))
except ValueError:
return grp.recipeupgrade_set.exclude(upgrade_type__in=['N','R']).order_by('-id').values_list('id', flat=True).first()
def __lt__(self, other):
if type(self.ver) == type(other.ver):
return self.ver < other.ver
elif isinstance(self.ver, str) and isinstance(other.ver, list):
return True
elif isinstance(self.ver, int) and isinstance(other.ver, int):
return False
def __repr__(self):
return repr(self.group)
class RecipeUpgradeDetail():
title = None
version = None
@ -735,9 +765,17 @@ class RecipeDetailView(DetailView):
context['maintainer_name'] = 'No maintainer'
details = []
multigroup = False
lastgroup = '' # can't use None here
for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by('group', '-commit_date', '-id'):
details.append(_get_recipe_upgrade_detail(maintplan, ru))
details.sort(key=lambda s: list(map(int, s.group.title.split('.') if s.group else [])), reverse=True)
if not multigroup:
if lastgroup == '':
lastgroup = ru.group
elif ru.group != lastgroup:
multigroup = True
details.sort(key=lambda s: RecipeUpgradeGroupSortItem(s.group), reverse=True)
context['multigroup'] = multigroup
context['recipe_upgrade_details'] = details
context['recipe_upgrade_detail_count'] = len(details)

2
templates/rrs/recipedetail.html
View File

@ -93,9 +93,11 @@
<th>Commit</th>
</tr>
{% for rud in recipe_upgrade_details %}
{% if multigroup %}
{% ifchanged rud.group %}
<tr><td colspan="6"><b>{{ rud.group.title }}</b></td></tr>
{% endifchanged %}
{% endif %}
<tr>
<td>{{ rud.title }}</td>
<td>{% if rud.upgrade_type != 'R' %}{{ rud.version }}{% if rud.upgrade_type == 'D' %} <span class="label label-warning">downgrade</span>{% endif %}{% endif %}</td>