From c0b84391827840ef302b1172f7533e887af50a54 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Tue, 26 Feb 2019 16:38:30 +1300 Subject: [PATCH] RRS: collect history independent of current recipes Recipes come and go over time, so when a recipe gets deleted the history for it goes away, which means that if you look back in time you do not see an accurate picture - you only see the subset of recipes that are currently present. Introduce an indirection between recipes and history that allows for old recipes to persist (mostly in name only). Signed-off-by: Paul Eggleton --- rrs/admin.py | 20 ++-- rrs/migrations/0020_recipesymbol_initial.py | 63 ++++++++++++ rrs/migrations/0021_recipesymbol_nonnull.py | 31 ++++++ rrs/migrations/0022_recipesymbol_finish.py | 27 +++++ rrs/models.py | 50 ++++++--- rrs/tools/rrs_maintainer_history.py | 32 +++--- rrs/tools/rrs_upgrade_history.py | 2 +- rrs/tools/rrs_upstream_history.py | 36 +++---- rrs/tools/upgrade_history_internal.py | 30 +++--- rrs/views.py | 107 +++++++++++--------- 10 files changed, 275 insertions(+), 123 deletions(-) create mode 100644 rrs/migrations/0020_recipesymbol_initial.py create mode 100644 rrs/migrations/0021_recipesymbol_nonnull.py create mode 100644 rrs/migrations/0022_recipesymbol_finish.py diff --git a/rrs/admin.py b/rrs/admin.py index 5f75026..5b47c92 100644 --- a/rrs/admin.py +++ b/rrs/admin.py @@ -15,7 +15,7 @@ from django.core.exceptions import ValidationError from rrs.models import Release, Milestone, Maintainer, RecipeMaintainerHistory, \ RecipeMaintainer, RecipeDistro, RecipeUpgrade, RecipeUpstream, \ RecipeUpstreamHistory, MaintenancePlan, MaintenancePlanLayerBranch, \ - RecipeMaintenanceLink + RecipeMaintenanceLink, RecipeSymbol class MaintenancePlanLayerBranchFormSet(BaseInlineFormSet): def __init__(self, *args, **kwargs): @@ -160,8 +160,8 @@ class RecipeMaintainerHistoryAdmin(admin.ModelAdmin): model = RecipeMaintainerHistory class RecipeMaintainerAdmin(admin.ModelAdmin): - search_fields = ['recipe__pn'] - list_filter = ['recipe__layerbranch__layer__name', 'history', 'maintainer__name'] + search_fields = ['recipesymbol__pn'] + list_filter = ['recipesymbol__layerbranch__layer__name', 'history', 'maintainer__name'] model = RecipeMaintainer class RecipeDistroAdmin(admin.ModelAdmin): @@ -170,8 +170,8 @@ class RecipeDistroAdmin(admin.ModelAdmin): model = RecipeDistro class RecipeUpgradeAdmin(admin.ModelAdmin): - search_fields = ['recipe__pn'] - list_filter = ['recipe__layerbranch__layer__name', + search_fields = ['recipesymbol__pn'] + list_filter = ['recipesymbol__layerbranch__layer__name', ('commit_date', DateFieldListFilter), 'maintainer__name'] model = RecipeUpgrade @@ -184,14 +184,19 @@ class RecipeUpstreamHistoryAdmin(admin.ModelAdmin): model = RecipeUpstreamHistory class RecipeUpstreamAdmin(admin.ModelAdmin): - search_fields = ['recipe__pn'] - list_filter = ['recipe__layerbranch__layer__name', 'status', + search_fields = ['recipesymbol__pn'] + list_filter = ['recipesymbol__layerbranch__layer__name', 'status', 'type', ('date', DateFieldListFilter), 'history'] model = RecipeUpstream class RecipeMaintenanceLinkAdmin(admin.ModelAdmin): model = RecipeMaintenanceLink +class RecipeSymbolAdmin(admin.ModelAdmin): + model = RecipeSymbol + search_fields = ['pn'] + list_filter = ['layerbranch'] + admin.site.register(MaintenancePlan, MaintenancePlanAdmin) admin.site.register(Release, ReleaseAdmin) admin.site.register(Milestone, MilestoneAdmin) @@ -203,3 +208,4 @@ admin.site.register(RecipeUpgrade, RecipeUpgradeAdmin) admin.site.register(RecipeUpstreamHistory, RecipeUpstreamHistoryAdmin) admin.site.register(RecipeUpstream, RecipeUpstreamAdmin) admin.site.register(RecipeMaintenanceLink, RecipeMaintenanceLinkAdmin) +admin.site.register(RecipeSymbol, RecipeSymbolAdmin) diff --git a/rrs/migrations/0020_recipesymbol_initial.py b/rrs/migrations/0020_recipesymbol_initial.py new file mode 100644 index 0000000..31a0126 --- /dev/null +++ b/rrs/migrations/0020_recipesymbol_initial.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2019-02-25 22:51 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +def populate_recipesymbol(apps, schema_editor): + RecipeSymbol = apps.get_model('rrs', 'RecipeSymbol') + RecipeMaintainer = apps.get_model('rrs', 'RecipeMaintainer') + RecipeUpgrade = apps.get_model('rrs', 'RecipeUpgrade') + RecipeUpstream = apps.get_model('rrs', 'RecipeUpstream') + + def populate_model(modelclass): + for obj in modelclass.objects.all(): + recipe = obj.recipe + rsym, created = RecipeSymbol.objects.get_or_create(layerbranch=recipe.layerbranch, pn=recipe.pn) + if created: + rsym.summary = recipe.summary + rsym.save() + obj.recipesymbol = rsym + obj.save() + + populate_model(RecipeMaintainer) + populate_model(RecipeUpgrade) + populate_model(RecipeUpstream) + + +class Migration(migrations.Migration): + + dependencies = [ + ('layerindex', '0027_patch_apply_order'), + ('rrs', '0019_maintplan_admin_delete_null'), + ] + + operations = [ + migrations.CreateModel( + name='RecipeSymbol', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pn', models.CharField(blank=True, max_length=100)), + ('summary', models.CharField(blank=True, max_length=200)), + ('layerbranch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.LayerBranch')), + ], + ), + migrations.AddField( + model_name='recipemaintainer', + name='recipesymbol', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'), + ), + migrations.AddField( + model_name='recipeupgrade', + name='recipesymbol', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'), + ), + migrations.AddField( + model_name='recipeupstream', + name='recipesymbol', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'), + ), + migrations.RunPython(populate_recipesymbol, reverse_code=migrations.RunPython.noop), + ] diff --git a/rrs/migrations/0021_recipesymbol_nonnull.py b/rrs/migrations/0021_recipesymbol_nonnull.py new file mode 100644 index 0000000..37518bb --- /dev/null +++ b/rrs/migrations/0021_recipesymbol_nonnull.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2019-02-26 02:53 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('rrs', '0020_recipesymbol_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='recipemaintainer', + name='recipesymbol', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'), + ), + migrations.AlterField( + model_name='recipeupgrade', + name='recipesymbol', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'), + ), + migrations.AlterField( + model_name='recipeupstream', + name='recipesymbol', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rrs.RecipeSymbol'), + ), + ] diff --git a/rrs/migrations/0022_recipesymbol_finish.py b/rrs/migrations/0022_recipesymbol_finish.py new file mode 100644 index 0000000..f9d9d37 --- /dev/null +++ b/rrs/migrations/0022_recipesymbol_finish.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2019-02-26 03:00 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('rrs', '0021_recipesymbol_nonnull'), + ] + + operations = [ + migrations.RemoveField( + model_name='recipemaintainer', + name='recipe', + ), + migrations.RemoveField( + model_name='recipeupgrade', + name='recipe', + ), + migrations.RemoveField( + model_name='recipeupstream', + name='recipe', + ), + ] diff --git a/rrs/models.py b/rrs/models.py index 2ff6f8e..3806662 100644 --- a/rrs/models.py +++ b/rrs/models.py @@ -193,6 +193,29 @@ class Milestone(models.Model): def __str__(self): return '%s: %s %s' % (self.release.plan.name, self.release.name, self.name) + +class RecipeSymbol(models.Model): + layerbranch = models.ForeignKey(LayerBranch) + pn = models.CharField(max_length=100, blank=True) + summary = models.CharField(max_length=200, blank=True) + + @staticmethod + def symbol(pn, layerbranch, summary=None): + rsym, created = RecipeSymbol.objects.get_or_create(pn=pn, layerbranch=layerbranch) + if created: + if summary: + rsym.summary = summary + else: + recipe = Recipe.objects.filter(pn=pn, layerbranch=layerbranch).first() + if recipe: + rsym.summary = recipe.summary + rsym.save() + return rsym + + def __str__(self): + return "%s: %s" % (str(self.layerbranch), self.pn) + + class Maintainer(models.Model): name = models.CharField(max_length=255, unique=True) email = models.CharField(max_length=255, blank=True) @@ -261,14 +284,14 @@ class RecipeMaintainerHistory(models.Model): return "%s: %s, %s" % (self.date, self.author.name, self.sha1[:10]) class RecipeMaintainer(models.Model): - recipe = models.ForeignKey(Recipe) + recipesymbol = models.ForeignKey(RecipeSymbol) maintainer = models.ForeignKey(Maintainer) history = models.ForeignKey(RecipeMaintainerHistory) @staticmethod def get_maintainer_by_recipe_and_history(recipe, history): - qry = RecipeMaintainer.objects.filter(recipe = recipe, - history = history) + qry = RecipeMaintainer.objects.filter(recipesymbol__pn=recipe.pn, + history=history) if qry: return qry[0].maintainer @@ -276,7 +299,7 @@ class RecipeMaintainer(models.Model): return None def __str__(self): - return "%s: %s <%s>" % (self.recipe.pn, self.maintainer.name, + return "%s: %s <%s>" % (self.recipesymbol.pn, self.maintainer.name, self.maintainer.email) class RecipeUpstreamHistory(models.Model): @@ -335,7 +358,7 @@ class RecipeUpstream(models.Model): ) RECIPE_UPSTREAM_TYPE_CHOICES_DICT = dict(RECIPE_UPSTREAM_TYPE_CHOICES) - recipe = models.ForeignKey(Recipe) + recipesymbol = models.ForeignKey(RecipeSymbol) history = models.ForeignKey(RecipeUpstreamHistory) version = models.CharField(max_length=100, blank=True) type = models.CharField(max_length=1, choices=RECIPE_UPSTREAM_TYPE_CHOICES, blank=True, db_index=True) @@ -373,8 +396,8 @@ class RecipeUpstream(models.Model): return qry @staticmethod - def get_by_recipe_and_history(recipe, history): - ru = RecipeUpstream.objects.filter(recipe = recipe, history = history) + def get_by_recipe_and_history(recipesymbol, history): + ru = RecipeUpstream.objects.filter(recipesymbol=recipesymbol, history=history) return ru[0] if ru else None def needs_upgrade(self): @@ -384,7 +407,7 @@ class RecipeUpstream(models.Model): return False def __str__(self): - return '%s: (%s, %s, %s)' % (self.recipe.pn, self.status, + return '%s: (%s, %s, %s)' % (self.recipesymbol.pn, self.status, self.version, self.date) class RecipeDistro(models.Model): @@ -407,7 +430,7 @@ class RecipeDistro(models.Model): class RecipeUpgrade(models.Model): - recipe = models.ForeignKey(Recipe) + recipesymbol = models.ForeignKey(RecipeSymbol) maintainer = models.ForeignKey(Maintainer, blank=True) sha1 = models.CharField(max_length=40, blank=True) title = models.CharField(max_length=1024, blank=True) @@ -417,7 +440,8 @@ class RecipeUpgrade(models.Model): @staticmethod def get_by_recipe_and_date(recipe, end_date): - ru = RecipeUpgrade.objects.filter(recipe = recipe, + ru = RecipeUpgrade.objects.filter(recipesymbol__pn=recipe.pn, + recipesymbol__layerbranch=recipe.layerbranch, commit_date__lte = end_date) return ru[len(ru) - 1] if ru else None @@ -425,10 +449,10 @@ class RecipeUpgrade(models.Model): return self.sha1[0:6] def commit_url(self): - return self.recipe.layerbranch.commit_url(self.sha1) + return self.recipesymbol.layerbranch.commit_url(self.sha1) def __str__(self): - return '%s: (%s, %s)' % (self.recipe.pn, self.version, + return '%s: (%s, %s)' % (self.recipesymbol.pn, self.version, self.commit_date) @@ -443,7 +467,7 @@ class RecipeMaintenanceLink(models.Model): if fnmatch.fnmatch(pn, rml.pn_match): recipe_link_objs = rmh.layerbranch.recipe_set.filter(pn=rml.pn_target) if recipe_link_objs: - lrm = RecipeMaintainer.objects.filter(recipe=recipe_link_objs[0], history=rmh) + lrm = RecipeMaintainer.objects.filter(recipesymbol__pn=recipe_link_objs[0].pn, history=rmh) if lrm: return lrm[0] return None diff --git a/rrs/tools/rrs_maintainer_history.py b/rrs/tools/rrs_maintainer_history.py index faaa7c0..b5cc510 100755 --- a/rrs/tools/rrs_maintainer_history.py +++ b/rrs/tools/rrs_maintainer_history.py @@ -22,7 +22,7 @@ from django.db import transaction import settings from layerindex.models import Recipe, LayerBranch, LayerItem -from rrs.models import MaintenancePlan, Maintainer, RecipeMaintainerHistory, RecipeMaintainer, RecipeMaintenanceLink +from rrs.models import MaintenancePlan, Maintainer, RecipeSymbol, RecipeMaintainerHistory, RecipeMaintainer, RecipeMaintenanceLink from django.core.exceptions import ObjectDoesNotExist # FIXME we shouldn't be hardcoded to expect RECIPE_MAINTAINER to be set in this file, @@ -110,28 +110,22 @@ def maintainers_inc_history(options, logger, maintplan, layerbranch, repodir, la res = get_recipe_maintainer(line, logger) if res: (pn, name, email) = res - qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) + m = Maintainer.create_or_update(name, email) - if qry: - m = Maintainer.create_or_update(name, email) + rm = RecipeMaintainer() + rm.recipesymbol = RecipeSymbol.symbol(pn, layerbranch) + rm.maintainer = m + rm.history = rms + rm.save() - rm = RecipeMaintainer() - rm.recipe = qry[0] - rm.maintainer = m - rm.history = rms - rm.save() - - logger.debug("%s: Change maintainer to %s in commit %s." % \ - (pn, m.name, commit)) - else: - logger.debug("%s: Not found in %s." % \ - (pn, layerbranch)) + logger.debug("%s: Change maintainer to %s in commit %s." % \ + (pn, m.name, commit)) # set missing recipes to no maintainer for recipe in layerbranch.recipe_set.all(): - if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): + if not RecipeMaintainer.objects.filter(recipesymbol__pn=recipe.pn, history=rms): rm = RecipeMaintainer() - rm.recipe = recipe + rm.recipesymbol = RecipeSymbol.symbol(recipe.pn, layerbranch, summary=recipe.summary) link_maintainer = RecipeMaintenanceLink.link_maintainer(recipe.pn, rms) if link_maintainer: rm.maintainer = link_maintainer.maintainer @@ -148,9 +142,9 @@ def maintainers_inc_history(options, logger, maintplan, layerbranch, repodir, la # set new recipes to no maintainer if don't have one rms = RecipeMaintainerHistory.get_last(layerbranch) for recipe in layerbranch.recipe_set.all(): - if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): + if not RecipeMaintainer.objects.filter(recipesymbol__pn=recipe.pn, history=rms): rm = RecipeMaintainer() - rm.recipe = recipe + rm.recipesymbol = RecipeSymbol.symbol(recipe.pn, layerbranch, summary=recipe.summary) link_maintainer = RecipeMaintenanceLink.link_maintainer(recipe.pn, rms) if link_maintainer: rm.maintainer = link_maintainer.maintainer diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py index aebf5aa..74e3e43 100755 --- a/rrs/tools/rrs_upgrade_history.py +++ b/rrs/tools/rrs_upgrade_history.py @@ -119,7 +119,7 @@ def upgrade_history(options, logger): for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(): layerbranch = maintplanbranch.layerbranch if options.fullreload and not options.dry_run: - RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete() + RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch).delete() layer = layerbranch.layer urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) diff --git a/rrs/tools/rrs_upstream_history.py b/rrs/tools/rrs_upstream_history.py index e9346a1..4451c15 100755 --- a/rrs/tools/rrs_upstream_history.py +++ b/rrs/tools/rrs_upstream_history.py @@ -35,7 +35,7 @@ bitbakepath = os.path.join(fetchdir, 'bitbake') sys.path.insert(0, os.path.join(bitbakepath, 'lib')) from layerindex.models import Recipe, LayerBranch -from rrs.models import RecipeUpstream, RecipeUpstreamHistory, MaintenancePlan +from rrs.models import RecipeUpstream, RecipeUpstreamHistory, MaintenancePlan, RecipeSymbol def set_regexes(d): """ @@ -86,15 +86,11 @@ def get_upstream_info(layerbranch, recipe_data, result): get_recipe_pv_without_srcpv pn = recipe_data.getVar('PN', True) - recipes = Recipe.objects.filter(layerbranch=layerbranch, pn=pn) - if not recipes: - logger.warning("%s: in layer branch %s not found." % \ - (pn, str(layerbranch))) - return - recipe = recipes[0] ru = RecipeUpstream() - ru.recipe = recipe + summary = recipe_data.getVar('SUMMARY', True) or recipe_data.getVar('DESCRIPTION', True) + ru.recipesymbol = RecipeSymbol.symbol(pn, layerbranch, summary=summary) + recipe_pv = recipe_data.getVar('PV', True) ru_info = None try: @@ -108,8 +104,8 @@ def get_upstream_info(layerbranch, recipe_data, result): ru.type = ru_info['type'] ru.date = ru_info['datetime'] - pv, _, _ = get_recipe_pv_without_srcpv(recipe.pv, - get_pv_type(recipe.pv)) + pv, _, _ = get_recipe_pv_without_srcpv(recipe_pv, + get_pv_type(recipe_pv)) upv, _, _ = get_recipe_pv_without_srcpv(ru_info['version'], get_pv_type(ru_info['version'])) @@ -122,7 +118,7 @@ def get_upstream_info(layerbranch, recipe_data, result): elif cmp_ver == 1: ru.status = 'D' # Downgrade, need to review why else: - logger.debug('Unable to determine upgrade status for %s: %s -> %s' % (recipe.pn, pv, upv)) + logger.debug('Unable to determine upgrade status for %s: %s -> %s' % (pn, pv, upv)) ru.status = 'U' # Unknown else: ru.version = '' @@ -150,6 +146,10 @@ if __name__=="__main__": help = "Do not write any data back to the database", action="store_true", dest="dry_run", default=False) + parser.add_option("--recipe", + help = "Recipe IDs to operate on", + action="store", dest="recipe", default=None) + options, args = parser.parse_args(sys.argv) logger.setLevel(options.loglevel) @@ -187,7 +187,11 @@ if __name__=="__main__": layerdir = os.path.join(repodir, layerbranch.vcs_subdir) recipe_files = [] - for recipe in layerbranch.recipe_set.all(): + if options.recipe: + recipe_qry = layerbranch.recipe_set.filter(id__in=options.recipe.split(',')) + else: + recipe_qry = layerbranch.recipe_set.all() + for recipe in recipe_qry: file = str(os.path.join(layerdir, recipe.full_path())) recipe_files.append(file) @@ -216,14 +220,12 @@ if __name__=="__main__": history.end_date = datetime.now() history.save() - for res in result: - (recipe, ru) = res - + logger.debug('Results for layerbranch %s:' % str(layerbranch)) + for ru in result: ru.history = history ru.save() - logger.debug('%s: layer branch %s, pv %s, upstream (%s)' % (recipe.pn, - str(layerbranch), recipe.pv, str(ru))) + logger.debug(str(ru)) finally: tinfoil.shutdown() diff --git a/rrs/tools/upgrade_history_internal.py b/rrs/tools/upgrade_history_internal.py index abcc2f2..02a0fbd 100644 --- a/rrs/tools/upgrade_history_internal.py +++ b/rrs/tools/upgrade_history_internal.py @@ -28,9 +28,9 @@ from layerindex.update_layer import split_recipe_fn """ Store upgrade into RecipeUpgrade model. """ -def _save_upgrade(recipe, pv, commit, title, info, logger): +def _save_upgrade(recipe_data, layerbranch, pv, commit, title, info, logger): from email.utils import parsedate_tz, mktime_tz - from rrs.models import Maintainer, RecipeUpgrade + from rrs.models import Maintainer, RecipeUpgrade, RecipeSymbol maintainer_name = info.split(';')[0] maintainer_email = info.split(';')[1] @@ -40,7 +40,8 @@ def _save_upgrade(recipe, pv, commit, title, info, logger): maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email) upgrade = RecipeUpgrade() - upgrade.recipe = recipe + summary = recipe_data.getVar('SUMMARY', True) or recipe_data.getVar('DESCRIPTION', True) + upgrade.recipesymbol = RecipeSymbol.symbol(recipe_data.getVar('PN', True), layerbranch, summary=summary) upgrade.maintainer = maintainer upgrade.author_date = datetime.utcfromtimestamp(mktime_tz( parsedate_tz(author_date))) @@ -55,8 +56,7 @@ def _save_upgrade(recipe, pv, commit, title, info, logger): Create upgrade receives new recipe_data and cmp versions. """ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False): - from layerindex.models import Recipe - from rrs.models import RecipeUpgrade + from rrs.models import RecipeUpgrade, RecipeSymbol from bb.utils import vercmp_string pn = recipe_data.getVar('PN', True) @@ -66,16 +66,11 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=F logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct)) return - recipes = Recipe.objects.filter(pn=pn, layerbranch=layerbranch).order_by('id') - if not recipes: - logger.warn("%s: Not found in Layer branch %s." % - (pn, str(layerbranch))) - return - recipe = recipes[0] + rsym = RecipeSymbol.objects.filter(pn=pn, layerbranch=layerbranch) try: latest_upgrade = RecipeUpgrade.objects.filter( - recipe = recipe).order_by('-commit_date')[0] + recipesymbol=rsym).order_by('-commit_date')[0] prev_pv = latest_upgrade.version except KeyboardInterrupt: raise @@ -83,8 +78,8 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=F prev_pv = None if prev_pv is None: - logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv)) - _save_upgrade(recipe, pv, ct, title, info, logger) + logger.debug("%s: Initial upgrade ( -> %s)." % (pn, pv)) + _save_upgrade(recipe_data, layerbranch, pv, ct, title, info, logger) else: from common import get_recipe_pv_without_srcpv @@ -95,18 +90,17 @@ def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=F try: if npv == 'git': - logger.debug("%s: Avoiding upgrade to unversioned git." % \ - (recipe.pn)) + logger.debug("%s: Avoiding upgrade to unversioned git." % pn) elif ppv == 'git' or vercmp_string(ppv, npv) == -1: if initial is True: logger.debug("%s: Update initial upgrade ( -> %s)." % \ - (recipe.pn, pv)) + (pn, pv)) latest_upgrade.version = pv latest_upgrade.save() else: logger.debug("%s: detected upgrade (%s -> %s)" \ " in ct %s." % (pn, prev_pv, pv, ct)) - _save_upgrade(recipe, pv, ct, title, info, logger) + _save_upgrade(recipe_data, layerbranch, pv, ct, title, info, logger) except KeyboardInterrupt: raise except Exception as e: diff --git a/rrs/views.py b/rrs/views.py index a16f53c..0d3986e 100644 --- a/rrs/views.py +++ b/rrs/views.py @@ -14,7 +14,7 @@ from django.contrib import messages from layerindex.models import Recipe, StaticBuildDep, Patch from rrs.models import Release, Milestone, Maintainer, RecipeMaintainerHistory, \ RecipeMaintainer, RecipeUpstreamHistory, RecipeUpstream, \ - RecipeDistro, RecipeUpgrade, MaintenancePlan + RecipeDistro, RecipeUpgrade, MaintenancePlan, RecipeSymbol @@ -83,7 +83,7 @@ class Raw(): recipes = [] cur = connection.cursor() - cur.execute("""SELECT DISTINCT rema.recipe_id + cur.execute("""SELECT DISTINCT rema.recipesymbol_id FROM rrs_recipemaintainer AS rema INNER JOIN rrs_maintainer AS ma ON rema.maintainer_id = ma.id @@ -101,12 +101,12 @@ class Raw(): stats = [] if date_id: - qry = """SELECT rema.recipe_id, ma.name + qry = """SELECT rema.recipesymbol_id, ma.name FROM rrs_recipemaintainer AS rema INNER JOIN rrs_maintainer AS ma ON rema.maintainer_id = ma.id WHERE rema.history_id = %s - AND rema.recipe_id IN %s;""" + AND rema.recipesymbol_id IN %s;""" cur = connection.cursor() cur.execute(qry, [str(date_id), tuple(recipes_id)]) stats = Raw.dictfetchall(cur) @@ -132,7 +132,7 @@ class Raw(): qry = """SELECT id, status, no_update_reason FROM rrs_recipeupstream WHERE history_id = %s - AND recipe_id IN %s;""" + AND recipesymbol_id IN %s;""" cur = connection.cursor() cur.execute(qry, [str(date_id.id), tuple(recipes)]) @@ -155,10 +155,10 @@ class Raw(): stats = [] if date_id: - qry = """SELECT recipe_id, status, no_update_reason, version + qry = """SELECT recipesymbol_id, status, no_update_reason, version FROM rrs_recipeupstream WHERE history_id = %s - AND recipe_id IN %s;""" + AND recipesymbol_id IN %s;""" cur = connection.cursor() cur.execute(qry, [str(date_id), tuple(recipes_id)]) stats = Raw.dictfetchall(cur) @@ -169,13 +169,13 @@ class Raw(): def get_reup_by_last_updated(layerbranch_id, date): """ Get last time the Recipes were upgraded """ cur = connection.cursor() - cur.execute("""SELECT recipe_id, MAX(commit_date) AS date + cur.execute("""SELECT recipesymbol_id, MAX(commit_date) AS date FROM rrs_recipeupgrade - INNER JOIN layerindex_recipe AS re - ON rrs_recipeupgrade.recipe_id = re.id + INNER JOIN rrs_recipesymbol AS rs + ON rrs_recipeupgrade.recipesymbol_id = rs.id WHERE commit_date <= %s - AND re.layerbranch_id = %s - GROUP BY recipe_id; + AND rs.layerbranch_id = %s + GROUP BY recipesymbol_id; """, [date, layerbranch_id]) return Raw.dictfetchall(cur) @@ -183,7 +183,7 @@ class Raw(): def get_reup_by_date(date_id): """ Get Recipes not up to date based on Recipe Upstream History """ cur = connection.cursor() - cur.execute("""SELECT DISTINCT recipe_id + cur.execute("""SELECT DISTINCT recipesymbol_id FROM rrs_recipeupstream WHERE status = 'N' AND history_id = %s @@ -194,18 +194,18 @@ class Raw(): def get_reupg_by_date(layerbranch_id, date): """ Get info for Recipes for the milestone """ cur = connection.cursor() - cur.execute("""SELECT re.id, re.pn, re.summary, te.version, rownum FROM ( - SELECT recipe_id, version, commit_date, ROW_NUMBER() OVER( - PARTITION BY recipe_id + cur.execute("""SELECT rs.id, rs.pn, rs.summary, te.version, rownum FROM ( + SELECT recipesymbol_id, version, commit_date, ROW_NUMBER() OVER( + PARTITION BY recipesymbol_id ORDER BY commit_date DESC ) AS rownum FROM rrs_recipeupgrade WHERE commit_date <= %s) AS te - INNER JOIN layerindex_recipe AS re - ON te.recipe_id = re.id + INNER JOIN rrs_recipesymbol AS rs + ON te.recipesymbol_id = rs.id WHERE rownum = 1 - AND re.layerbranch_id = %s - ORDER BY re.pn; + AND rs.layerbranch_id = %s + ORDER BY rs.pn; """, [date, layerbranch_id]) return Raw.dictfetchall(cur) @@ -213,11 +213,11 @@ class Raw(): def get_reupg_by_dates_and_recipes(start_date, end_date, recipes_id): """ Get Recipe Upgrade for the milestone based on Recipes """ cur = connection.cursor() - qry = """SELECT DISTINCT recipe_id + qry = """SELECT DISTINCT recipesymbol_id FROM rrs_recipeupgrade WHERE commit_date >= %s AND commit_date <= %s - AND recipe_id IN %s;""" + AND recipesymbol_id IN %s;""" cur.execute(qry, [start_date, end_date, tuple(recipes_id)]) return Raw.dictfetchall(cur) @@ -408,33 +408,33 @@ def _get_recipe_list(milestone): recipe_last_updated = Raw.get_reup_by_last_updated( layerbranch.id, milestone.end_date) for rela in recipe_last_updated: - recipe_last_updated_dict_all[rela['recipe_id']] = rela + recipe_last_updated_dict_all[rela['recipesymbol_id']] = rela if recipe_upstream_history: recipe_upstream_all = Raw.get_reup_by_recipes_and_date( recipes_ids, recipe_upstream_history.id) for reup in recipe_upstream_all: - recipe_upstream_dict_all[reup['recipe_id']] = reup + recipe_upstream_dict_all[reup['recipesymbol_id']] = reup if recipe_maintainer_history: maintainers_all = Raw.get_ma_by_recipes_and_date( recipes_ids, recipe_maintainer_history[0]) for ma in maintainers_all: - maintainers_dict_all[ma['recipe_id']] = ma['name'] + maintainers_dict_all[ma['recipesymbol_id']] = ma['name'] for recipe in recipes: upstream_version = '' upstream_status = '' no_update_reason = '' outdated = '' + recipesymbol = RecipeSymbol.objects.filter(id=recipe['id']).first() if recipe_upstream_history: recipe_upstream = recipe_upstream_dict_all.get(recipe['id']) if not recipe_upstream: - recipe_add = Recipe.objects.filter(id = recipe['id'])[0] recipe_upstream_add = RecipeUpstream() recipe_upstream_add.history = recipe_upstream_history - recipe_upstream_add.recipe = recipe_add + recipe_upstream_add.recipesymbol = recipesymbol recipe_upstream_add.version = '' recipe_upstream_add.type = 'M' # Manual recipe_upstream_add.status = 'U' # Unknown @@ -469,9 +469,14 @@ def _get_recipe_list(milestone): recipe_list_item.upstream_status = upstream_status recipe_list_item.upstream_version = upstream_version recipe_list_item.outdated = outdated - patches = Patch.objects.filter(recipe__id=recipe['id']) - recipe_list_item.patches_total = patches.count() - recipe_list_item.patches_pending = patches.filter(status='P').count() + recipeobj = recipesymbol.layerbranch.recipe_set.filter(pn=recipesymbol.pn, layerbranch=recipesymbol.layerbranch).first() + if recipeobj: + patches = recipeobj.patch_set + recipe_list_item.patches_total = patches.count() + recipe_list_item.patches_pending = patches.filter(status='P').count() + else: + recipe_list_item.patches_total = 0 + recipe_list_item.patches_pending = 0 recipe_list_item.maintainer_name = maintainer_name recipe_list_item.no_update_reason = no_update_reason recipe_list.append(recipe_list_item) @@ -643,7 +648,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade): if milestone: milestone_name = milestone.name recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date( - recipe_upgrade.recipe.layerbranch, + recipe_upgrade.recipesymbol.layerbranch, milestone.end_date) is_recipe_maintainer = False @@ -659,7 +664,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade): commit_date = recipe_upgrade.commit_date.date().isoformat() commit = recipe_upgrade.sha1[:10] - commit_url = recipe_upgrade.recipe.layerbranch.commit_url(recipe_upgrade.sha1) + commit_url = recipe_upgrade.recipesymbol.layerbranch.commit_url(recipe_upgrade.sha1) rud = RecipeUpgradeDetail(recipe_upgrade.title, recipe_upgrade.version, \ maintplan.name, release_name, milestone_name, commit_date, maintainer_name, \ @@ -668,7 +673,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade): return rud class RecipeDetailView(DetailView): - model = Recipe + model = RecipeSymbol def get_queryset(self): self.maintplan_name = self.kwargs['maintplan_name'] @@ -676,10 +681,13 @@ class RecipeDetailView(DetailView): def get_context_data(self, **kwargs): context = super(RecipeDetailView, self).get_context_data(**kwargs) - recipe = self.get_object() - if not recipe: + recipesymbol = self.get_object() + if not recipesymbol: raise django.http.Http404 + recipe = recipesymbol.layerbranch.recipe_set.filter(pn=recipesymbol.pn, layerbranch=recipesymbol.layerbranch).last() + context['recipe'] = recipe + maintplan = get_object_or_404(MaintenancePlan, name=self.maintplan_name) context['maintplan_name'] = maintplan.name context['maintplan'] = maintplan @@ -692,13 +700,13 @@ class RecipeDetailView(DetailView): context['upstream_version'] = '' context['upstream_no_update_reason'] = '' recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range( - recipe.layerbranch, + recipesymbol.layerbranch, milestone.start_date, milestone.end_date ) if recipe_upstream_history: recipe_upstream = RecipeUpstream.get_by_recipe_and_history( - recipe, recipe_upstream_history) + recipesymbol, recipe_upstream_history) if recipe_upstream: if recipe_upstream.status == 'N' and recipe_upstream.no_update_reason: recipe_upstream.status = 'C' @@ -709,9 +717,9 @@ class RecipeDetailView(DetailView): context['upstream_version'] = recipe_upstream.version context['upstream_no_update_reason'] = recipe_upstream.no_update_reason - self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(recipe.layerbranch) - recipe_maintainer = RecipeMaintainer.objects.filter(recipe = recipe, - history = self.recipe_maintainer_history) + self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(recipesymbol.layerbranch) + recipe_maintainer = RecipeMaintainer.objects.filter(recipesymbol=recipesymbol, + history=self.recipe_maintainer_history) if recipe_maintainer: maintainer = recipe_maintainer[0].maintainer context['maintainer_name'] = maintainer.name @@ -719,23 +727,26 @@ class RecipeDetailView(DetailView): context['maintainer_name'] = 'No maintainer' context['recipe_upgrade_details'] = [] - for ru in RecipeUpgrade.objects.filter(recipe = - recipe).order_by('-commit_date'): + for ru in RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).order_by('-commit_date'): context['recipe_upgrade_details'].append(_get_recipe_upgrade_detail(maintplan, ru)) context['recipe_upgrade_detail_count'] = len(context['recipe_upgrade_details']) context['recipe_layer_branch_url'] = _get_layer_branch_url( - recipe.layerbranch.branch.name, recipe.layerbranch.layer.name) + recipesymbol.layerbranch.branch.name, recipesymbol.layerbranch.layer.name) context['recipe_provides'] = [] - for p in recipe.provides.split(): - context['recipe_provides'].append(p) + if recipe: + for p in recipe.provides.split(): + context['recipe_provides'].append(p) - context['recipe_depends'] = StaticBuildDep.objects.filter(recipes__id=recipe.id).values_list('name', flat=True) + context['recipe_depends'] = StaticBuildDep.objects.filter(recipes__id=recipe.id).values_list('name', flat=True) - context['recipe_distros'] = RecipeDistro.get_distros_by_recipe(recipe) + context['recipe_distros'] = RecipeDistro.get_distros_by_recipe(recipe) + else: + context['recipe_depends'] = [] + context['recipe_distros'] = [] - context['otherbranch_recipes'] = Recipe.objects.filter(layerbranch__layer=recipe.layerbranch.layer, layerbranch__branch__comparison=False, pn=recipe.pn).order_by('layerbranch__branch__sort_priority') + context['otherbranch_recipes'] = Recipe.objects.filter(layerbranch__layer=recipesymbol.layerbranch.layer, layerbranch__branch__comparison=False, pn=recipesymbol.pn).order_by('layerbranch__branch__sort_priority') return context