MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-19 20:59:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

RRS: collect history independent of current recipes

Browse Source 
Recipes come and go over time, so when a recipe gets deleted the history
for it goes away, which means that if you look back in time you do not
see an accurate picture - you only see the subset of recipes that are
currently present. Introduce an indirection between recipes and history
that allows for old recipes to persist (mostly in name only).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2019-02-26 16:38:30 +13:00
parent 50fc6780e0
commit c0b8439182
10 changed files with 275 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
rrs/admin.py
View File

@ -15,7 +15,7 @@ from django.core.exceptions import ValidationError
from rrs.models import Release, Milestone, Maintainer, RecipeMaintainerHistory, \ from rrs.models import Release, Milestone, Maintainer, RecipeMaintainerHistory, \
RecipeMaintainer, RecipeDistro, RecipeUpgrade, RecipeUpstream, \ RecipeMaintainer, RecipeDistro, RecipeUpgrade, RecipeUpstream, \
RecipeUpstreamHistory, MaintenancePlan, MaintenancePlanLayerBranch, \ RecipeUpstreamHistory, MaintenancePlan, MaintenancePlanLayerBranch, \
RecipeMaintenanceLink RecipeMaintenanceLink, RecipeSymbol
class MaintenancePlanLayerBranchFormSet(BaseInlineFormSet): class MaintenancePlanLayerBranchFormSet(BaseInlineFormSet):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
@ -160,8 +160,8 @@ class RecipeMaintainerHistoryAdmin(admin.ModelAdmin):
model = RecipeMaintainerHistory model = RecipeMaintainerHistory
class RecipeMaintainerAdmin(admin.ModelAdmin): class RecipeMaintainerAdmin(admin.ModelAdmin):
search_fields = ['recipe__pn'] search_fields = ['recipesymbol__pn']
list_filter = ['recipe__layerbranch__layer__name', 'history', 'maintainer__name'] list_filter = ['recipesymbol__layerbranch__layer__name', 'history', 'maintainer__name']
model = RecipeMaintainer model = RecipeMaintainer
class RecipeDistroAdmin(admin.ModelAdmin): class RecipeDistroAdmin(admin.ModelAdmin):
@ -170,8 +170,8 @@ class RecipeDistroAdmin(admin.ModelAdmin):
model = RecipeDistro model = RecipeDistro
class RecipeUpgradeAdmin(admin.ModelAdmin): class RecipeUpgradeAdmin(admin.ModelAdmin):
search_fields = ['recipe__pn'] search_fields = ['recipesymbol__pn']
list_filter = ['recipe__layerbranch__layer__name', list_filter = ['recipesymbol__layerbranch__layer__name',
('commit_date', DateFieldListFilter), 'maintainer__name'] ('commit_date', DateFieldListFilter), 'maintainer__name']
model = RecipeUpgrade model = RecipeUpgrade
@ -184,14 +184,19 @@ class RecipeUpstreamHistoryAdmin(admin.ModelAdmin):
model = RecipeUpstreamHistory model = RecipeUpstreamHistory
class RecipeUpstreamAdmin(admin.ModelAdmin): class RecipeUpstreamAdmin(admin.ModelAdmin):
search_fields = ['recipe__pn'] search_fields = ['recipesymbol__pn']
list_filter = ['recipe__layerbranch__layer__name', 'status', list_filter = ['recipesymbol__layerbranch__layer__name', 'status',
'type', ('date', DateFieldListFilter), 'history'] 'type', ('date', DateFieldListFilter), 'history']
model = RecipeUpstream model = RecipeUpstream
class RecipeMaintenanceLinkAdmin(admin.ModelAdmin): class RecipeMaintenanceLinkAdmin(admin.ModelAdmin):
model = RecipeMaintenanceLink model = RecipeMaintenanceLink
class RecipeSymbolAdmin(admin.ModelAdmin):
model = RecipeSymbol
search_fields = ['pn']
list_filter = ['layerbranch']
admin.site.register(MaintenancePlan, MaintenancePlanAdmin) admin.site.register(MaintenancePlan, MaintenancePlanAdmin)
admin.site.register(Release, ReleaseAdmin) admin.site.register(Release, ReleaseAdmin)
admin.site.register(Milestone, MilestoneAdmin) admin.site.register(Milestone, MilestoneAdmin)
@ -203,3 +208,4 @@ admin.site.register(RecipeUpgrade, RecipeUpgradeAdmin)
admin.site.register(RecipeUpstreamHistory, RecipeUpstreamHistoryAdmin) admin.site.register(RecipeUpstreamHistory, RecipeUpstreamHistoryAdmin)
admin.site.register(RecipeUpstream, RecipeUpstreamAdmin) admin.site.register(RecipeUpstream, RecipeUpstreamAdmin)
admin.site.register(RecipeMaintenanceLink, RecipeMaintenanceLinkAdmin) admin.site.register(RecipeMaintenanceLink, RecipeMaintenanceLinkAdmin)
admin.site.register(RecipeSymbol, RecipeSymbolAdmin)

63
rrs/migrations/0020_recipesymbol_initial.py Normal file
View File

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2019-02-25 22:51
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
def populate_recipesymbol(apps, schema_editor):
RecipeSymbol = apps.get_model('rrs', 'RecipeSymbol')
RecipeMaintainer = apps.get_model('rrs', 'RecipeMaintainer')
RecipeUpgrade = apps.get_model('rrs', 'RecipeUpgrade')
RecipeUpstream = apps.get_model('rrs', 'RecipeUpstream')
def populate_model(modelclass):
for obj in modelclass.objects.all():
recipe = obj.recipe
rsym, created = RecipeSymbol.objects.get_or_create(layerbranch=recipe.layerbranch, pn=recipe.pn)
if created:
rsym.summary = recipe.summary
rsym.save()
obj.recipesymbol = rsym
obj.save()
populate_model(RecipeMaintainer)
populate_model(RecipeUpgrade)
populate_model(RecipeUpstream)
class Migration(migrations.Migration):
dependencies = [
('layerindex', '0027_patch_apply_order'),
('rrs', '0019_maintplan_admin_delete_null'),
]
operations = [
migrations.CreateModel(
name='RecipeSymbol',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('pn', models.CharField(blank=True, max_length=100)),
('summary', models.CharField(blank=True, max_length=200)),
('layerbranch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.LayerBranch')),
],
),
migrations.AddField(
model_name='recipemaintainer',
name='recipesymbol',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'),
),
migrations.AddField(
model_name='recipeupgrade',
name='recipesymbol',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'),
),
migrations.AddField(
model_name='recipeupstream',
name='recipesymbol',
field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'),
),
migrations.RunPython(populate_recipesymbol, reverse_code=migrations.RunPython.noop),
]

31
rrs/migrations/0021_recipesymbol_nonnull.py Normal file
View File

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2019-02-26 02:53
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('rrs', '0020_recipesymbol_initial'),
]
operations = [
migrations.AlterField(
model_name='recipemaintainer',
name='recipesymbol',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'),
),
migrations.AlterField(
model_name='recipeupgrade',
name='recipesymbol',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'),
),
migrations.AlterField(
model_name='recipeupstream',
name='recipesymbol',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'),
),
]

27
rrs/migrations/0022_recipesymbol_finish.py Normal file
View File

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.16 on 2019-02-26 03:00
from __future__ import unicode_literals
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('rrs', '0021_recipesymbol_nonnull'),
]
operations = [
migrations.RemoveField(
model_name='recipemaintainer',
name='recipe',
),
migrations.RemoveField(
model_name='recipeupgrade',
name='recipe',
),
migrations.RemoveField(
model_name='recipeupstream',
name='recipe',
),
]

50
rrs/models.py
View File

@ -193,6 +193,29 @@ class Milestone(models.Model):
def __str__(self): def __str__(self):
return '%s: %s %s' % (self.release.plan.name, self.release.name, self.name) return '%s: %s %s' % (self.release.plan.name, self.release.name, self.name)
class RecipeSymbol(models.Model):
layerbranch = models.ForeignKey(LayerBranch)
pn = models.CharField(max_length=100, blank=True)
summary = models.CharField(max_length=200, blank=True)
@staticmethod
def symbol(pn, layerbranch, summary=None):
rsym, created = RecipeSymbol.objects.get_or_create(pn=pn, layerbranch=layerbranch)
if created:
if summary:
rsym.summary = summary
else:
recipe = Recipe.objects.filter(pn=pn, layerbranch=layerbranch).first()
if recipe:
rsym.summary = recipe.summary
rsym.save()
return rsym
def __str__(self):
return "%s: %s" % (str(self.layerbranch), self.pn)
class Maintainer(models.Model): class Maintainer(models.Model):
name = models.CharField(max_length=255, unique=True) name = models.CharField(max_length=255, unique=True)
email = models.CharField(max_length=255, blank=True) email = models.CharField(max_length=255, blank=True)
@ -261,14 +284,14 @@ class RecipeMaintainerHistory(models.Model):
return "%s: %s, %s" % (self.date, self.author.name, self.sha1[:10]) return "%s: %s, %s" % (self.date, self.author.name, self.sha1[:10])
class RecipeMaintainer(models.Model): class RecipeMaintainer(models.Model):
recipe = models.ForeignKey(Recipe) recipesymbol = models.ForeignKey(RecipeSymbol)
maintainer = models.ForeignKey(Maintainer) maintainer = models.ForeignKey(Maintainer)
history = models.ForeignKey(RecipeMaintainerHistory) history = models.ForeignKey(RecipeMaintainerHistory)
@staticmethod @staticmethod
def get_maintainer_by_recipe_and_history(recipe, history): def get_maintainer_by_recipe_and_history(recipe, history):
qry = RecipeMaintainer.objects.filter(recipe = recipe, qry = RecipeMaintainer.objects.filter(recipesymbol__pn=recipe.pn,
history = history) history=history)
if qry: if qry:
return qry[0].maintainer return qry[0].maintainer
@ -276,7 +299,7 @@ class RecipeMaintainer(models.Model):
return None return None
def __str__(self): def __str__(self):
return "%s: %s <%s>" % (self.recipe.pn, self.maintainer.name, return "%s: %s <%s>" % (self.recipesymbol.pn, self.maintainer.name,
self.maintainer.email) self.maintainer.email)
class RecipeUpstreamHistory(models.Model): class RecipeUpstreamHistory(models.Model):
@ -335,7 +358,7 @@ class RecipeUpstream(models.Model):
) )
RECIPE_UPSTREAM_TYPE_CHOICES_DICT = dict(RECIPE_UPSTREAM_TYPE_CHOICES) RECIPE_UPSTREAM_TYPE_CHOICES_DICT = dict(RECIPE_UPSTREAM_TYPE_CHOICES)
recipe = models.ForeignKey(Recipe) recipesymbol = models.ForeignKey(RecipeSymbol)
history = models.ForeignKey(RecipeUpstreamHistory) history = models.ForeignKey(RecipeUpstreamHistory)
version = models.CharField(max_length=100, blank=True) version = models.CharField(max_length=100, blank=True)
type = models.CharField(max_length=1, choices=RECIPE_UPSTREAM_TYPE_CHOICES, blank=True, db_index=True) type = models.CharField(max_length=1, choices=RECIPE_UPSTREAM_TYPE_CHOICES, blank=True, db_index=True)
@ -373,8 +396,8 @@ class RecipeUpstream(models.Model):
return qry return qry
@staticmethod @staticmethod
def get_by_recipe_and_history(recipe, history): def get_by_recipe_and_history(recipesymbol, history):
ru = RecipeUpstream.objects.filter(recipe = recipe, history = history) ru = RecipeUpstream.objects.filter(recipesymbol=recipesymbol, history=history)
return ru[0] if ru else None return ru[0] if ru else None
def needs_upgrade(self): def needs_upgrade(self):
@ -384,7 +407,7 @@ class RecipeUpstream(models.Model):
return False return False
def __str__(self): def __str__(self):
return '%s: (%s, %s, %s)' % (self.recipe.pn, self.status, return '%s: (%s, %s, %s)' % (self.recipesymbol.pn, self.status,
self.version, self.date) self.version, self.date)
class RecipeDistro(models.Model): class RecipeDistro(models.Model):
@ -407,7 +430,7 @@ class RecipeDistro(models.Model):
class RecipeUpgrade(models.Model): class RecipeUpgrade(models.Model):
recipe = models.ForeignKey(Recipe) recipesymbol = models.ForeignKey(RecipeSymbol)
maintainer = models.ForeignKey(Maintainer, blank=True) maintainer = models.ForeignKey(Maintainer, blank=True)
sha1 = models.CharField(max_length=40, blank=True) sha1 = models.CharField(max_length=40, blank=True)
title = models.CharField(max_length=1024, blank=True) title = models.CharField(max_length=1024, blank=True)
@ -417,7 +440,8 @@ class RecipeUpgrade(models.Model):
@staticmethod @staticmethod
def get_by_recipe_and_date(recipe, end_date): def get_by_recipe_and_date(recipe, end_date):
ru = RecipeUpgrade.objects.filter(recipe = recipe, ru = RecipeUpgrade.objects.filter(recipesymbol__pn=recipe.pn,
recipesymbol__layerbranch=recipe.layerbranch,
commit_date__lte = end_date) commit_date__lte = end_date)
return ru[len(ru) - 1] if ru else None return ru[len(ru) - 1] if ru else None
@ -425,10 +449,10 @@ class RecipeUpgrade(models.Model):
return self.sha1[0:6] return self.sha1[0:6]
def commit_url(self): def commit_url(self):
return self.recipe.layerbranch.commit_url(self.sha1) return self.recipesymbol.layerbranch.commit_url(self.sha1)
def __str__(self): def __str__(self):
return '%s: (%s, %s)' % (self.recipe.pn, self.version, return '%s: (%s, %s)' % (self.recipesymbol.pn, self.version,
self.commit_date) self.commit_date)
@ -443,7 +467,7 @@ class RecipeMaintenanceLink(models.Model):
if fnmatch.fnmatch(pn, rml.pn_match): if fnmatch.fnmatch(pn, rml.pn_match):
recipe_link_objs = rmh.layerbranch.recipe_set.filter(pn=rml.pn_target) recipe_link_objs = rmh.layerbranch.recipe_set.filter(pn=rml.pn_target)
if recipe_link_objs: if recipe_link_objs:
lrm = RecipeMaintainer.objects.filter(recipe=recipe_link_objs[0], history=rmh) lrm = RecipeMaintainer.objects.filter(recipesymbol__pn=recipe_link_objs[0].pn, history=rmh)
if lrm: if lrm:
return lrm[0] return lrm[0]
return None return None

32
rrs/tools/rrs_maintainer_history.py
View File

@ -22,7 +22,7 @@ from django.db import transaction
import settings import settings
from layerindex.models import Recipe, LayerBranch, LayerItem from layerindex.models import Recipe, LayerBranch, LayerItem
from rrs.models import MaintenancePlan, Maintainer, RecipeMaintainerHistory, RecipeMaintainer, RecipeMaintenanceLink from rrs.models import MaintenancePlan, Maintainer, RecipeSymbol, RecipeMaintainerHistory, RecipeMaintainer, RecipeMaintenanceLink
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
# FIXME we shouldn't be hardcoded to expect RECIPE_MAINTAINER to be set in this file, # FIXME we shouldn't be hardcoded to expect RECIPE_MAINTAINER to be set in this file,
@ -110,28 +110,22 @@ def maintainers_inc_history(options, logger, maintplan, layerbranch, repodir, la
res = get_recipe_maintainer(line, logger) res = get_recipe_maintainer(line, logger)
if res: if res:
(pn, name, email) = res (pn, name, email) = res
qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) m = Maintainer.create_or_update(name, email)
if qry: rm = RecipeMaintainer()
m = Maintainer.create_or_update(name, email) rm.recipesymbol = RecipeSymbol.symbol(pn, layerbranch)
rm.maintainer = m
rm.history = rms
rm.save()
rm = RecipeMaintainer() logger.debug("%s: Change maintainer to %s in commit %s." % \
rm.recipe = qry[0] (pn, m.name, commit))
rm.maintainer = m
rm.history = rms
rm.save()
logger.debug("%s: Change maintainer to %s in commit %s." % \
(pn, m.name, commit))
else:
logger.debug("%s: Not found in %s." % \
(pn, layerbranch))
# set missing recipes to no maintainer # set missing recipes to no maintainer
for recipe in layerbranch.recipe_set.all(): for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): if not RecipeMaintainer.objects.filter(recipesymbol__pn=recipe.pn, history=rms):
rm = RecipeMaintainer() rm = RecipeMaintainer()
rm.recipe = recipe rm.recipesymbol = RecipeSymbol.symbol(recipe.pn, layerbranch, summary=recipe.summary)
link_maintainer = RecipeMaintenanceLink.link_maintainer(recipe.pn, rms) link_maintainer = RecipeMaintenanceLink.link_maintainer(recipe.pn, rms)
if link_maintainer: if link_maintainer:
rm.maintainer = link_maintainer.maintainer rm.maintainer = link_maintainer.maintainer
@ -148,9 +142,9 @@ def maintainers_inc_history(options, logger, maintplan, layerbranch, repodir, la
# set new recipes to no maintainer if don't have one # set new recipes to no maintainer if don't have one
rms = RecipeMaintainerHistory.get_last(layerbranch) rms = RecipeMaintainerHistory.get_last(layerbranch)
for recipe in layerbranch.recipe_set.all(): for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): if not RecipeMaintainer.objects.filter(recipesymbol__pn=recipe.pn, history=rms):
rm = RecipeMaintainer() rm = RecipeMaintainer()
rm.recipe = recipe rm.recipesymbol = RecipeSymbol.symbol(recipe.pn, layerbranch, summary=recipe.summary)
link_maintainer = RecipeMaintenanceLink.link_maintainer(recipe.pn, rms) link_maintainer = RecipeMaintenanceLink.link_maintainer(recipe.pn, rms)
if link_maintainer: if link_maintainer:
rm.maintainer = link_maintainer.maintainer rm.maintainer = link_maintainer.maintainer

2
rrs/tools/rrs_upgrade_history.py
View File

@ -119,7 +119,7 @@ def upgrade_history(options, logger):
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(): for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanbranch.layerbranch layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run: if options.fullreload and not options.dry_run:
RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete() RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch).delete()
layer = layerbranch.layer layer = layerbranch.layer
urldir = layer.get_fetch_dir() urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir) repodir = os.path.join(fetchdir, urldir)

36
rrs/tools/rrs_upstream_history.py
View File

@ -35,7 +35,7 @@ bitbakepath = os.path.join(fetchdir, 'bitbake')
sys.path.insert(0, os.path.join(bitbakepath, 'lib')) sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
from layerindex.models import Recipe, LayerBranch from layerindex.models import Recipe, LayerBranch
from rrs.models import RecipeUpstream, RecipeUpstreamHistory, MaintenancePlan from rrs.models import RecipeUpstream, RecipeUpstreamHistory, MaintenancePlan, RecipeSymbol
def set_regexes(d): def set_regexes(d):
""" """
@ -86,15 +86,11 @@ def get_upstream_info(layerbranch, recipe_data, result):
get_recipe_pv_without_srcpv get_recipe_pv_without_srcpv
pn = recipe_data.getVar('PN', True) pn = recipe_data.getVar('PN', True)
recipes = Recipe.objects.filter(layerbranch=layerbranch, pn=pn)
if not recipes:
logger.warning("%s: in layer branch %s not found." % \
(pn, str(layerbranch)))
return
recipe = recipes[0]
ru = RecipeUpstream() ru = RecipeUpstream()
ru.recipe = recipe summary = recipe_data.getVar('SUMMARY', True) or recipe_data.getVar('DESCRIPTION', True)
ru.recipesymbol = RecipeSymbol.symbol(pn, layerbranch, summary=summary)
recipe_pv = recipe_data.getVar('PV', True)
ru_info = None ru_info = None
try: try:
@ -108,8 +104,8 @@ def get_upstream_info(layerbranch, recipe_data, result):
ru.type = ru_info['type'] ru.type = ru_info['type']
ru.date = ru_info['datetime'] ru.date = ru_info['datetime']
pv, _, _ = get_recipe_pv_without_srcpv(recipe.pv, pv, _, _ = get_recipe_pv_without_srcpv(recipe_pv,
get_pv_type(recipe.pv)) get_pv_type(recipe_pv))
upv, _, _ = get_recipe_pv_without_srcpv(ru_info['version'], upv, _, _ = get_recipe_pv_without_srcpv(ru_info['version'],
get_pv_type(ru_info['version'])) get_pv_type(ru_info['version']))
@ -122,7 +118,7 @@ def get_upstream_info(layerbranch, recipe_data, result):
elif cmp_ver == 1: elif cmp_ver == 1:
ru.status = 'D' # Downgrade, need to review why ru.status = 'D' # Downgrade, need to review why
else: else:
logger.debug('Unable to determine upgrade status for %s: %s -> %s' % (recipe.pn, pv, upv)) logger.debug('Unable to determine upgrade status for %s: %s -> %s' % (pn, pv, upv))
ru.status = 'U' # Unknown ru.status = 'U' # Unknown
else: else:
ru.version = '' ru.version = ''
@ -150,6 +146,10 @@ if __name__=="__main__":
help = "Do not write any data back to the database", help = "Do not write any data back to the database",
action="store_true", dest="dry_run", default=False) action="store_true", dest="dry_run", default=False)
parser.add_option("--recipe",
help = "Recipe IDs to operate on",
action="store", dest="recipe", default=None)
options, args = parser.parse_args(sys.argv) options, args = parser.parse_args(sys.argv)
logger.setLevel(options.loglevel) logger.setLevel(options.loglevel)
@ -187,7 +187,11 @@ if __name__=="__main__":
layerdir = os.path.join(repodir, layerbranch.vcs_subdir) layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
recipe_files = [] recipe_files = []
for recipe in layerbranch.recipe_set.all(): if options.recipe:
recipe_qry = layerbranch.recipe_set.filter(id__in=options.recipe.split(','))
else:
recipe_qry = layerbranch.recipe_set.all()
for recipe in recipe_qry:
file = str(os.path.join(layerdir, recipe.full_path())) file = str(os.path.join(layerdir, recipe.full_path()))
recipe_files.append(file) recipe_files.append(file)
@ -216,14 +220,12 @@ if __name__=="__main__":
history.end_date = datetime.now() history.end_date = datetime.now()
history.save() history.save()
for res in result: logger.debug('Results for layerbranch %s:' % str(layerbranch))
(recipe, ru) = res for ru in result:
ru.history = history ru.history = history
ru.save() ru.save()
logger.debug('%s: layer branch %s, pv %s, upstream (%s)' % (recipe.pn, logger.debug(str(ru))
str(layerbranch), recipe.pv, str(ru)))
finally: finally:
tinfoil.shutdown() tinfoil.shutdown()

30
rrs/tools/upgrade_history_internal.py
View File

@ -28,9 +28,9 @@ from layerindex.update_layer import split_recipe_fn
""" """
Store upgrade into RecipeUpgrade model. Store upgrade into RecipeUpgrade model.
""" """
def _save_upgrade(recipe, pv, commit, title, info, logger): def _save_upgrade(recipe_data, layerbranch, pv, commit, title, info, logger):
from email.utils import parsedate_tz, mktime_tz from email.utils import parsedate_tz, mktime_tz
from rrs.models import Maintainer, RecipeUpgrade from rrs.models import Maintainer, RecipeUpgrade, RecipeSymbol
maintainer_name = info.split(';')[0] maintainer_name = info.split(';')[0]
maintainer_email = info.split(';')[1] maintainer_email = info.split(';')[1]
@ -40,7 +40,8 @@ def _save_upgrade(recipe, pv, commit, title, info, logger):
maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email)
upgrade = RecipeUpgrade() upgrade = RecipeUpgrade()
upgrade.recipe = recipe summary = recipe_data.getVar('SUMMARY', True) or recipe_data.getVar('DESCRIPTION', True)
upgrade.recipesymbol = RecipeSymbol.symbol(recipe_data.getVar('PN', True), layerbranch, summary=summary)
upgrade.maintainer = maintainer upgrade.maintainer = maintainer
upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( upgrade.author_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(author_date))) parsedate_tz(author_date)))
@ -55,8 +56,7 @@ def _save_upgrade(recipe, pv, commit, title, info, logger):
Create upgrade receives new recipe_data and cmp versions. Create upgrade receives new recipe_data and cmp versions.
""" """
def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False):
from layerindex.models import Recipe from rrs.models import RecipeUpgrade, RecipeSymbol
from rrs.models import RecipeUpgrade
from bb.utils import vercmp_string from bb.utils import vercmp_string
pn = recipe_data.getVar('PN', True) pn = recipe_data.getVar('PN', True)
@ -66,16 +66,11 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=F
logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct)) logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct))
return return
recipes = Recipe.objects.filter(pn=pn, layerbranch=layerbranch).order_by('id') rsym = RecipeSymbol.objects.filter(pn=pn, layerbranch=layerbranch)
if not recipes:
logger.warn("%s: Not found in Layer branch %s." %
(pn, str(layerbranch)))
return
recipe = recipes[0]
try: try:
latest_upgrade = RecipeUpgrade.objects.filter( latest_upgrade = RecipeUpgrade.objects.filter(
recipe = recipe).order_by('-commit_date')[0] recipesymbol=rsym).order_by('-commit_date')[0]
prev_pv = latest_upgrade.version prev_pv = latest_upgrade.version
except KeyboardInterrupt: except KeyboardInterrupt:
raise raise
@ -83,8 +78,8 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=F
prev_pv = None prev_pv = None
if prev_pv is None: if prev_pv is None:
logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) logger.debug("%s: Initial upgrade ( -> %s)." % (pn, pv))
_save_upgrade(recipe, pv, ct, title, info, logger) _save_upgrade(recipe_data, layerbranch, pv, ct, title, info, logger)
else: else:
from common import get_recipe_pv_without_srcpv from common import get_recipe_pv_without_srcpv
@ -95,18 +90,17 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=F
try: try:
if npv == 'git': if npv == 'git':
logger.debug("%s: Avoiding upgrade to unversioned git." % \ logger.debug("%s: Avoiding upgrade to unversioned git." % pn)
(recipe.pn))
elif ppv == 'git' or vercmp_string(ppv, npv) == -1: elif ppv == 'git' or vercmp_string(ppv, npv) == -1:
if initial is True: if initial is True:
logger.debug("%s: Update initial upgrade ( -> %s)." % \ logger.debug("%s: Update initial upgrade ( -> %s)." % \
(recipe.pn, pv)) (pn, pv))
latest_upgrade.version = pv latest_upgrade.version = pv
latest_upgrade.save() latest_upgrade.save()
else: else:
logger.debug("%s: detected upgrade (%s -> %s)" \ logger.debug("%s: detected upgrade (%s -> %s)" \
" in ct %s." % (pn, prev_pv, pv, ct)) " in ct %s." % (pn, prev_pv, pv, ct))
_save_upgrade(recipe, pv, ct, title, info, logger) _save_upgrade(recipe_data, layerbranch, pv, ct, title, info, logger)
except KeyboardInterrupt: except KeyboardInterrupt:
raise raise
except Exception as e: except Exception as e:

107
rrs/views.py
View File

@ -14,7 +14,7 @@ from django.contrib import messages
from layerindex.models import Recipe, StaticBuildDep, Patch from layerindex.models import Recipe, StaticBuildDep, Patch
from rrs.models import Release, Milestone, Maintainer, RecipeMaintainerHistory, \ from rrs.models import Release, Milestone, Maintainer, RecipeMaintainerHistory, \
RecipeMaintainer, RecipeUpstreamHistory, RecipeUpstream, \ RecipeMaintainer, RecipeUpstreamHistory, RecipeUpstream, \
RecipeDistro, RecipeUpgrade, MaintenancePlan RecipeDistro, RecipeUpgrade, MaintenancePlan, RecipeSymbol
@ -83,7 +83,7 @@ class Raw():
recipes = [] recipes = []
cur = connection.cursor() cur = connection.cursor()
cur.execute("""SELECT DISTINCT rema.recipe_id cur.execute("""SELECT DISTINCT rema.recipesymbol_id
FROM rrs_recipemaintainer AS rema FROM rrs_recipemaintainer AS rema
INNER JOIN rrs_maintainer AS ma INNER JOIN rrs_maintainer AS ma
ON rema.maintainer_id = ma.id ON rema.maintainer_id = ma.id
@ -101,12 +101,12 @@ class Raw():
stats = [] stats = []
if date_id: if date_id:
qry = """SELECT rema.recipe_id, ma.name qry = """SELECT rema.recipesymbol_id, ma.name
FROM rrs_recipemaintainer AS rema FROM rrs_recipemaintainer AS rema
INNER JOIN rrs_maintainer AS ma INNER JOIN rrs_maintainer AS ma
ON rema.maintainer_id = ma.id ON rema.maintainer_id = ma.id
WHERE rema.history_id = %s WHERE rema.history_id = %s
AND rema.recipe_id IN %s;""" AND rema.recipesymbol_id IN %s;"""
cur = connection.cursor() cur = connection.cursor()
cur.execute(qry, [str(date_id), tuple(recipes_id)]) cur.execute(qry, [str(date_id), tuple(recipes_id)])
stats = Raw.dictfetchall(cur) stats = Raw.dictfetchall(cur)
@ -132,7 +132,7 @@ class Raw():
qry = """SELECT id, status, no_update_reason qry = """SELECT id, status, no_update_reason
FROM rrs_recipeupstream FROM rrs_recipeupstream
WHERE history_id = %s WHERE history_id = %s
AND recipe_id IN %s;""" AND recipesymbol_id IN %s;"""
cur = connection.cursor() cur = connection.cursor()
cur.execute(qry, [str(date_id.id), tuple(recipes)]) cur.execute(qry, [str(date_id.id), tuple(recipes)])
@ -155,10 +155,10 @@ class Raw():
stats = [] stats = []
if date_id: if date_id:
qry = """SELECT recipe_id, status, no_update_reason, version qry = """SELECT recipesymbol_id, status, no_update_reason, version
FROM rrs_recipeupstream FROM rrs_recipeupstream
WHERE history_id = %s WHERE history_id = %s
AND recipe_id IN %s;""" AND recipesymbol_id IN %s;"""
cur = connection.cursor() cur = connection.cursor()
cur.execute(qry, [str(date_id), tuple(recipes_id)]) cur.execute(qry, [str(date_id), tuple(recipes_id)])
stats = Raw.dictfetchall(cur) stats = Raw.dictfetchall(cur)
@ -169,13 +169,13 @@ class Raw():
def get_reup_by_last_updated(layerbranch_id, date): def get_reup_by_last_updated(layerbranch_id, date):
""" Get last time the Recipes were upgraded """ """ Get last time the Recipes were upgraded """
cur = connection.cursor() cur = connection.cursor()
cur.execute("""SELECT recipe_id, MAX(commit_date) AS date cur.execute("""SELECT recipesymbol_id, MAX(commit_date) AS date
FROM rrs_recipeupgrade FROM rrs_recipeupgrade
INNER JOIN layerindex_recipe AS re INNER JOIN rrs_recipesymbol AS rs
ON rrs_recipeupgrade.recipe_id = re.id ON rrs_recipeupgrade.recipesymbol_id = rs.id
WHERE commit_date <= %s WHERE commit_date <= %s
AND re.layerbranch_id = %s AND rs.layerbranch_id = %s
GROUP BY recipe_id; GROUP BY recipesymbol_id;
""", [date, layerbranch_id]) """, [date, layerbranch_id])
return Raw.dictfetchall(cur) return Raw.dictfetchall(cur)
@ -183,7 +183,7 @@ class Raw():
def get_reup_by_date(date_id): def get_reup_by_date(date_id):
""" Get Recipes not up to date based on Recipe Upstream History """ """ Get Recipes not up to date based on Recipe Upstream History """
cur = connection.cursor() cur = connection.cursor()
cur.execute("""SELECT DISTINCT recipe_id cur.execute("""SELECT DISTINCT recipesymbol_id
FROM rrs_recipeupstream FROM rrs_recipeupstream
WHERE status = 'N' WHERE status = 'N'
AND history_id = %s AND history_id = %s
@ -194,18 +194,18 @@ class Raw():
def get_reupg_by_date(layerbranch_id, date): def get_reupg_by_date(layerbranch_id, date):
""" Get info for Recipes for the milestone """ """ Get info for Recipes for the milestone """
cur = connection.cursor() cur = connection.cursor()
cur.execute("""SELECT re.id, re.pn, re.summary, te.version, rownum FROM ( cur.execute("""SELECT rs.id, rs.pn, rs.summary, te.version, rownum FROM (
SELECT recipe_id, version, commit_date, ROW_NUMBER() OVER( SELECT recipesymbol_id, version, commit_date, ROW_NUMBER() OVER(
PARTITION BY recipe_id PARTITION BY recipesymbol_id
ORDER BY commit_date DESC ORDER BY commit_date DESC
) AS rownum ) AS rownum
FROM rrs_recipeupgrade FROM rrs_recipeupgrade
WHERE commit_date <= %s) AS te WHERE commit_date <= %s) AS te
INNER JOIN layerindex_recipe AS re INNER JOIN rrs_recipesymbol AS rs
ON te.recipe_id = re.id ON te.recipesymbol_id = rs.id
WHERE rownum = 1 WHERE rownum = 1
AND re.layerbranch_id = %s AND rs.layerbranch_id = %s
ORDER BY re.pn; ORDER BY rs.pn;
""", [date, layerbranch_id]) """, [date, layerbranch_id])
return Raw.dictfetchall(cur) return Raw.dictfetchall(cur)
@ -213,11 +213,11 @@ class Raw():
def get_reupg_by_dates_and_recipes(start_date, end_date, recipes_id): def get_reupg_by_dates_and_recipes(start_date, end_date, recipes_id):
""" Get Recipe Upgrade for the milestone based on Recipes """ """ Get Recipe Upgrade for the milestone based on Recipes """
cur = connection.cursor() cur = connection.cursor()
qry = """SELECT DISTINCT recipe_id qry = """SELECT DISTINCT recipesymbol_id
FROM rrs_recipeupgrade FROM rrs_recipeupgrade
WHERE commit_date >= %s WHERE commit_date >= %s
AND commit_date <= %s AND commit_date <= %s
AND recipe_id IN %s;""" AND recipesymbol_id IN %s;"""
cur.execute(qry, [start_date, end_date, tuple(recipes_id)]) cur.execute(qry, [start_date, end_date, tuple(recipes_id)])
return Raw.dictfetchall(cur) return Raw.dictfetchall(cur)
@ -408,33 +408,33 @@ def _get_recipe_list(milestone):
recipe_last_updated = Raw.get_reup_by_last_updated( recipe_last_updated = Raw.get_reup_by_last_updated(
layerbranch.id, milestone.end_date) layerbranch.id, milestone.end_date)
for rela in recipe_last_updated: for rela in recipe_last_updated:
recipe_last_updated_dict_all[rela['recipe_id']] = rela recipe_last_updated_dict_all[rela['recipesymbol_id']] = rela
if recipe_upstream_history: if recipe_upstream_history:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date( recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipes_ids, recipe_upstream_history.id) recipes_ids, recipe_upstream_history.id)
for reup in recipe_upstream_all: for reup in recipe_upstream_all:
recipe_upstream_dict_all[reup['recipe_id']] = reup recipe_upstream_dict_all[reup['recipesymbol_id']] = reup
if recipe_maintainer_history: if recipe_maintainer_history:
maintainers_all = Raw.get_ma_by_recipes_and_date( maintainers_all = Raw.get_ma_by_recipes_and_date(
recipes_ids, recipe_maintainer_history[0]) recipes_ids, recipe_maintainer_history[0])
for ma in maintainers_all: for ma in maintainers_all:
maintainers_dict_all[ma['recipe_id']] = ma['name'] maintainers_dict_all[ma['recipesymbol_id']] = ma['name']
for recipe in recipes: for recipe in recipes:
upstream_version = '' upstream_version = ''
upstream_status = '' upstream_status = ''
no_update_reason = '' no_update_reason = ''
outdated = '' outdated = ''
recipesymbol = RecipeSymbol.objects.filter(id=recipe['id']).first()
if recipe_upstream_history: if recipe_upstream_history:
recipe_upstream = recipe_upstream_dict_all.get(recipe['id']) recipe_upstream = recipe_upstream_dict_all.get(recipe['id'])
if not recipe_upstream: if not recipe_upstream:
recipe_add = Recipe.objects.filter(id = recipe['id'])[0]
recipe_upstream_add = RecipeUpstream() recipe_upstream_add = RecipeUpstream()
recipe_upstream_add.history = recipe_upstream_history recipe_upstream_add.history = recipe_upstream_history
recipe_upstream_add.recipe = recipe_add recipe_upstream_add.recipesymbol = recipesymbol
recipe_upstream_add.version = '' recipe_upstream_add.version = ''
recipe_upstream_add.type = 'M' # Manual recipe_upstream_add.type = 'M' # Manual
recipe_upstream_add.status = 'U' # Unknown recipe_upstream_add.status = 'U' # Unknown
@ -469,9 +469,14 @@ def _get_recipe_list(milestone):
recipe_list_item.upstream_status = upstream_status recipe_list_item.upstream_status = upstream_status
recipe_list_item.upstream_version = upstream_version recipe_list_item.upstream_version = upstream_version
recipe_list_item.outdated = outdated recipe_list_item.outdated = outdated
patches = Patch.objects.filter(recipe__id=recipe['id']) recipeobj = recipesymbol.layerbranch.recipe_set.filter(pn=recipesymbol.pn, layerbranch=recipesymbol.layerbranch).first()
recipe_list_item.patches_total = patches.count() if recipeobj:
recipe_list_item.patches_pending = patches.filter(status='P').count() patches = recipeobj.patch_set
recipe_list_item.patches_total = patches.count()
recipe_list_item.patches_pending = patches.filter(status='P').count()
else:
recipe_list_item.patches_total = 0
recipe_list_item.patches_pending = 0
recipe_list_item.maintainer_name = maintainer_name recipe_list_item.maintainer_name = maintainer_name
recipe_list_item.no_update_reason = no_update_reason recipe_list_item.no_update_reason = no_update_reason
recipe_list.append(recipe_list_item) recipe_list.append(recipe_list_item)
@ -643,7 +648,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade):
if milestone: if milestone:
milestone_name = milestone.name milestone_name = milestone.name
recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date( recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date(
recipe_upgrade.recipe.layerbranch, recipe_upgrade.recipesymbol.layerbranch,
milestone.end_date) milestone.end_date)
is_recipe_maintainer = False is_recipe_maintainer = False
@ -659,7 +664,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade):
commit_date = recipe_upgrade.commit_date.date().isoformat() commit_date = recipe_upgrade.commit_date.date().isoformat()
commit = recipe_upgrade.sha1[:10] commit = recipe_upgrade.sha1[:10]
commit_url = recipe_upgrade.recipe.layerbranch.commit_url(recipe_upgrade.sha1) commit_url = recipe_upgrade.recipesymbol.layerbranch.commit_url(recipe_upgrade.sha1)
rud = RecipeUpgradeDetail(recipe_upgrade.title, recipe_upgrade.version, \ rud = RecipeUpgradeDetail(recipe_upgrade.title, recipe_upgrade.version, \
maintplan.name, release_name, milestone_name, commit_date, maintainer_name, \ maintplan.name, release_name, milestone_name, commit_date, maintainer_name, \
@ -668,7 +673,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade):
return rud return rud
class RecipeDetailView(DetailView): class RecipeDetailView(DetailView):
model = Recipe model = RecipeSymbol
def get_queryset(self): def get_queryset(self):
self.maintplan_name = self.kwargs['maintplan_name'] self.maintplan_name = self.kwargs['maintplan_name']
@ -676,10 +681,13 @@ class RecipeDetailView(DetailView):
def get_context_data(self, **kwargs): def get_context_data(self, **kwargs):
context = super(RecipeDetailView, self).get_context_data(**kwargs) context = super(RecipeDetailView, self).get_context_data(**kwargs)
recipe = self.get_object() recipesymbol = self.get_object()
if not recipe: if not recipesymbol:
raise django.http.Http404 raise django.http.Http404
recipe = recipesymbol.layerbranch.recipe_set.filter(pn=recipesymbol.pn, layerbranch=recipesymbol.layerbranch).last()
context['recipe'] = recipe
maintplan = get_object_or_404(MaintenancePlan, name=self.maintplan_name) maintplan = get_object_or_404(MaintenancePlan, name=self.maintplan_name)
context['maintplan_name'] = maintplan.name context['maintplan_name'] = maintplan.name
context['maintplan'] = maintplan context['maintplan'] = maintplan
@ -692,13 +700,13 @@ class RecipeDetailView(DetailView):
context['upstream_version'] = '' context['upstream_version'] = ''
context['upstream_no_update_reason'] = '' context['upstream_no_update_reason'] = ''
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range( recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
recipe.layerbranch, recipesymbol.layerbranch,
milestone.start_date, milestone.start_date,
milestone.end_date milestone.end_date
) )
if recipe_upstream_history: if recipe_upstream_history:
recipe_upstream = RecipeUpstream.get_by_recipe_and_history( recipe_upstream = RecipeUpstream.get_by_recipe_and_history(
recipe, recipe_upstream_history) recipesymbol, recipe_upstream_history)
if recipe_upstream: if recipe_upstream:
if recipe_upstream.status == 'N' and recipe_upstream.no_update_reason: if recipe_upstream.status == 'N' and recipe_upstream.no_update_reason:
recipe_upstream.status = 'C' recipe_upstream.status = 'C'
@ -709,9 +717,9 @@ class RecipeDetailView(DetailView):
context['upstream_version'] = recipe_upstream.version context['upstream_version'] = recipe_upstream.version
context['upstream_no_update_reason'] = recipe_upstream.no_update_reason context['upstream_no_update_reason'] = recipe_upstream.no_update_reason
self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(recipe.layerbranch) self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(recipesymbol.layerbranch)
recipe_maintainer = RecipeMaintainer.objects.filter(recipe = recipe, recipe_maintainer = RecipeMaintainer.objects.filter(recipesymbol=recipesymbol,
history = self.recipe_maintainer_history) history=self.recipe_maintainer_history)
if recipe_maintainer: if recipe_maintainer:
maintainer = recipe_maintainer[0].maintainer maintainer = recipe_maintainer[0].maintainer
context['maintainer_name'] = maintainer.name context['maintainer_name'] = maintainer.name
@ -719,23 +727,26 @@ class RecipeDetailView(DetailView):
context['maintainer_name'] = 'No maintainer' context['maintainer_name'] = 'No maintainer'
context['recipe_upgrade_details'] = [] context['recipe_upgrade_details'] = []
for ru in RecipeUpgrade.objects.filter(recipe = for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).order_by('-commit_date'):
recipe).order_by('-commit_date'):
context['recipe_upgrade_details'].append(_get_recipe_upgrade_detail(maintplan, ru)) context['recipe_upgrade_details'].append(_get_recipe_upgrade_detail(maintplan, ru))
context['recipe_upgrade_detail_count'] = len(context['recipe_upgrade_details']) context['recipe_upgrade_detail_count'] = len(context['recipe_upgrade_details'])
context['recipe_layer_branch_url'] = _get_layer_branch_url( context['recipe_layer_branch_url'] = _get_layer_branch_url(
recipe.layerbranch.branch.name, recipe.layerbranch.layer.name) recipesymbol.layerbranch.branch.name, recipesymbol.layerbranch.layer.name)
context['recipe_provides'] = [] context['recipe_provides'] = []
for p in recipe.provides.split(): if recipe:
context['recipe_provides'].append(p) for p in recipe.provides.split():
context['recipe_provides'].append(p)
context['recipe_depends'] = StaticBuildDep.objects.filter(recipes__id=recipe.id).values_list('name', flat=True) context['recipe_depends'] = StaticBuildDep.objects.filter(recipes__id=recipe.id).values_list('name', flat=True)
context['recipe_distros'] = RecipeDistro.get_distros_by_recipe(recipe) context['recipe_distros'] = RecipeDistro.get_distros_by_recipe(recipe)
else:
context['recipe_depends'] = []
context['recipe_distros'] = []
context['otherbranch_recipes'] = Recipe.objects.filter(layerbranch__layer=recipe.layerbranch.layer, layerbranch__branch__comparison=False, pn=recipe.pn).order_by('layerbranch__branch__sort_priority') context['otherbranch_recipes'] = Recipe.objects.filter(layerbranch__layer=recipesymbol.layerbranch.layer, layerbranch__branch__comparison=False, pn=recipesymbol.pn).order_by('layerbranch__branch__sort_priority')
return context return context