rrs: link maintenance/upstream history to layerbranch

RecipeUpstreamHistory was not linked to the layer it was produced from,
which meant that it wasn't easy to query for a different maintenance
plan (i.e. a different layer) and thus the maintenance plan selection
on the recipe list didn't really work. Add a link field, populate it in
a migration and then make it required.

We had added a link earlier from RecipeMaintainerHistory to LayerBranch
but it was optional; for the same reasons we now populate it and make it
required.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2018-04-11 11:01:33 +12:00
parent da3bfff3a1
commit 7606664eeb
11 changed files with 303 additions and 147 deletions

View File

@ -87,7 +87,7 @@ class MaintainerAdmin(admin.ModelAdmin):
class RecipeMaintainerHistoryAdmin(admin.ModelAdmin):
search_fields = ['title', 'author__name', 'sha1']
list_filter = ['author__name', ('date', DateFieldListFilter)]
list_filter = ['layerbranch__layer', 'author__name', ('date', DateFieldListFilter)]
model = RecipeMaintainerHistory
class RecipeMaintainerAdmin(admin.ModelAdmin):
@ -108,6 +108,7 @@ class RecipeUpgradeAdmin(admin.ModelAdmin):
class RecipeUpstreamHistoryAdmin(admin.ModelAdmin):
list_filter = [
'layerbranch__layer',
('start_date', DateFieldListFilter),
('end_date', DateFieldListFilter)
]

View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rrs', '0011_release_name_unique'),
]
operations = [
migrations.AddField(
model_name='recipeupstreamhistory',
name='layerbranch',
field=models.ForeignKey(null=True, to='layerindex.LayerBranch'),
),
]

View File

@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import settings
def populate_layerbranch(apps, schema_editor):
RecipeUpstreamHistory = apps.get_model('rrs', 'RecipeUpstreamHistory')
LayerBranch = apps.get_model('layerindex', 'LayerBranch')
if not settings.CORE_LAYER_NAME:
raise Exception('Please set CORE_LAYER_NAME in settings.py')
core_layerbranch = LayerBranch.objects.filter(layer__name=settings.CORE_LAYER_NAME).first()
if not core_layerbranch:
raise Exception('Unable to find core layer "%s" specified in CORE_LAYER_NAME in settings.py' % settings.CORE_LAYER_NAME)
for row in RecipeUpstreamHistory.objects.all():
row.layerbranch = core_layerbranch
row.save()
class Migration(migrations.Migration):
dependencies = [
('rrs', '0012_reup_layerbranch_field'),
]
operations = [
migrations.RunPython(populate_layerbranch, reverse_code=migrations.RunPython.noop),
]

View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rrs', '0013_reup_layerbranch_populate'),
]
operations = [
migrations.AlterField(
model_name='recipeupstreamhistory',
name='layerbranch',
field=models.ForeignKey(to='layerindex.LayerBranch'),
),
]

View File

@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import settings
def populate_rmh_layerbranch(apps, schema_editor):
RecipeMaintainerHistory = apps.get_model('rrs', 'RecipeMaintainerHistory')
LayerBranch = apps.get_model('layerindex', 'LayerBranch')
if not settings.CORE_LAYER_NAME:
raise Exception('Please set CORE_LAYER_NAME in settings.py')
core_layerbranch = LayerBranch.objects.filter(layer__name=settings.CORE_LAYER_NAME).first()
if not core_layerbranch:
raise Exception('Unable to find core layer "%s" specified in CORE_LAYER_NAME in settings.py - please set up the layerindex application first' % settings.CORE_LAYER_NAME)
for row in RecipeMaintainerHistory.objects.all():
row.layerbranch = core_layerbranch
row.save()
class Migration(migrations.Migration):
dependencies = [
('rrs', '0014_reup_layerbranch_nonnull'),
]
operations = [
migrations.RunPython(populate_rmh_layerbranch, reverse_code=migrations.RunPython.noop),
]

View File

@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rrs', '0015_rmh_layerbranch_populate'),
]
operations = [
migrations.AlterField(
model_name='recipemaintainerhistory',
name='layerbranch',
field=models.ForeignKey(to='layerindex.LayerBranch'),
),
]

View File

@ -218,11 +218,11 @@ class RecipeMaintainerHistory(models.Model):
date = models.DateTimeField(db_index=True)
author = models.ForeignKey(Maintainer)
sha1 = models.CharField(max_length=64, unique=True)
layerbranch = models.ForeignKey(LayerBranch, blank=True, null=True)
layerbranch = models.ForeignKey(LayerBranch)
@staticmethod
def get_last():
rmh_qry = RecipeMaintainerHistory.objects.filter().order_by('-date')
def get_last(layerbranch):
rmh_qry = RecipeMaintainerHistory.objects.filter(layerbranch=layerbranch).order_by('-date')
if rmh_qry:
return rmh_qry[0]
@ -230,14 +230,16 @@ class RecipeMaintainerHistory(models.Model):
return None
@staticmethod
def get_by_end_date(end_date):
def get_by_end_date(layerbranch, end_date):
rmh_qry = RecipeMaintainerHistory.objects.filter(
layerbranch=layerbranch,
date__lte = end_date).order_by('-date')
if rmh_qry:
return rmh_qry[0]
rmh_qry = RecipeMaintainerHistory.objects.filter(
layerbranch=layerbranch
).order_by('date')
if rmh_qry:
return rmh_qry[0]
@ -267,12 +269,14 @@ class RecipeMaintainer(models.Model):
self.maintainer.email)
class RecipeUpstreamHistory(models.Model):
layerbranch = models.ForeignKey(LayerBranch)
start_date = models.DateTimeField(db_index=True)
end_date = models.DateTimeField(db_index=True)
@staticmethod
def get_last_by_date_range(start, end):
history = RecipeUpstreamHistory.objects.filter(start_date__gte = start,
def get_last_by_date_range(layerbranch, start, end):
history = RecipeUpstreamHistory.objects.filter(layerbranch=layerbranch,
start_date__gte = start,
start_date__lte = end).order_by('-start_date')
if history:
@ -281,8 +285,9 @@ class RecipeUpstreamHistory(models.Model):
return None
@staticmethod
def get_first_by_date_range(start, end):
history = RecipeUpstreamHistory.objects.filter(start_date__gte = start,
def get_first_by_date_range(layerbranch, start, end):
history = RecipeUpstreamHistory.objects.filter(layerbranch=layerbranch,
start_date__gte = start,
start_date__lte = end).order_by('start_date')
if history:
@ -291,8 +296,8 @@ class RecipeUpstreamHistory(models.Model):
return None
@staticmethod
def get_last():
history = RecipeUpstreamHistory.objects.filter().order_by('-start_date')
def get_last(layerbranch):
history = RecipeUpstreamHistory.objects.filter(layerbranch=layerbranch).order_by('-start_date')
if history:
return history[0]

View File

@ -112,7 +112,7 @@ def maintainer_history(options, logger):
try:
with transaction.atomic():
for commit in commits.strip().split("\n"):
if RecipeMaintainerHistory.objects.filter(sha1=commit):
if RecipeMaintainerHistory.objects.filter(layerbranch=layerbranch, sha1=commit):
continue
logger.debug("Analysing commit %s ..." % (commit))
@ -170,7 +170,7 @@ def maintainer_history(options, logger):
(recipe.pn, rms.sha1))
# set new recipes to no maintainer if don't have one
rms = RecipeMaintainerHistory.get_last()
rms = RecipeMaintainerHistory.get_last(layerbranch)
for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
rm = RecipeMaintainer()

View File

@ -159,12 +159,12 @@ def main():
for item in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = item.layerbranch
recipe_upstream_history = RecipeUpstreamHistory.get_last()
recipe_upstream_history = RecipeUpstreamHistory.get_last(layerbranch)
if recipe_upstream_history is None:
logger.warn('I don\'t have Upstream information yet, run update.py script')
sys.exit(1)
recipe_maintainer_history = RecipeMaintainerHistory.get_last()
recipe_maintainer_history = RecipeMaintainerHistory.get_last(layerbranch)
if recipe_maintainer_history is None:
logger.warn('I don\'t have Maintainership information yet,' +
' run rrs_maintainer_history.py script')

View File

@ -199,7 +199,7 @@ if __name__=="__main__":
for recipe_data in recipes:
set_regexes(recipe_data)
history = RecipeUpstreamHistory(start_date = datetime.now())
history = RecipeUpstreamHistory(layerbranch=layerbranch, start_date=datetime.now())
result = []
for recipe_data in recipes:

View File

@ -114,7 +114,7 @@ class Raw():
return stats
@staticmethod
def get_reup_statistics(date, date_id):
def get_reup_statistics(maintplan, date, date_id):
""" Special case to get recipes statistics removing gcc-source duplicates """
recipes = []
updated = 0
@ -122,11 +122,13 @@ class Raw():
cant = 0
unknown = 0
all_recipes = Raw.get_reupg_by_date(date)
for re in all_recipes:
for maintplanlayer in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanlayer.layerbranch
layerbranch_recipes = Raw.get_reupg_by_date(layerbranch.id, date)
for re in layerbranch_recipes:
recipes.append(re["id"])
if date_id:
if date_id and recipes:
recipes = str(recipes).strip('[]')
qry = """SELECT id, status, no_update_reason
FROM rrs_recipeupstream"""
@ -166,14 +168,17 @@ class Raw():
return stats
@staticmethod
def get_reup_by_last_updated(date):
def get_reup_by_last_updated(layerbranch_id, date):
""" Get last time the Recipes were upgraded """
cur = connection.cursor()
cur.execute("""SELECT recipe_id, MAX(commit_date) AS date
FROM rrs_recipeupgrade
INNER JOIN layerindex_recipe AS re
ON rrs_recipeupgrade.recipe_id = re.id
WHERE commit_date <= %s
AND re.layerbranch_id = %s
GROUP BY recipe_id;
""", [date])
""", [date, layerbranch_id])
return Raw.dictfetchall(cur)
@staticmethod
@ -188,7 +193,7 @@ class Raw():
return [i[0] for i in cur.fetchall()]
@staticmethod
def get_reupg_by_date(date):
def get_reupg_by_date(layerbranch_id, date):
""" Get info for Recipes for the milestone """
cur = connection.cursor()
cur.execute("""SELECT re.id, re.pn, re.summary, te.version, rownum FROM (
@ -201,8 +206,9 @@ class Raw():
INNER JOIN layerindex_recipe AS re
ON te.recipe_id = re.id
WHERE rownum = 1
AND re.layerbranch_id = %s
ORDER BY re.pn;
""", [date])
""", [date, layerbranch_id])
return Raw.dictfetchall(cur)
@staticmethod
@ -220,25 +226,27 @@ class Raw():
return Raw.dictfetchall(cur)
@staticmethod
def get_remahi_by_end_date(date):
def get_remahi_by_end_date(layerbranch_id, date):
""" Get the latest Recipe Maintainer History for the milestone """
cur = connection.cursor()
cur.execute("""SELECT id
FROM rrs_recipemaintainerhistory
WHERE date <= %s
AND layerbranch_id = %s
ORDER BY date DESC
LIMIT 1;
""", [str(date)])
""", [str(date), layerbranch_id])
ret = cur.fetchone()
if not ret:
cur.execute("""SELECT id
FROM rrs_recipemaintainerhistory
WHERE layerbranch_id = %s
ORDER BY date
LIMIT 1;
""")
""", [layerbranch_id])
ret = cur.fetchone()
return ret
@ -256,32 +264,40 @@ class Raw():
def _get_milestone_statistics(milestone, maintainer_name=None):
milestone_statistics = {}
milestone_statistics['all'] = 0
milestone_statistics['up_to_date'] = 0
milestone_statistics['not_updated'] = 0
milestone_statistics['cant_be_updated'] = 0
milestone_statistics['unknown'] = 0
if maintainer_name is None:
milestone_statistics['all_upgraded'] = 0
milestone_statistics['all_not_upgraded'] = 0
for maintplanlayer in milestone.release.plan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanlayer.layerbranch
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
layerbranch,
milestone.start_date,
milestone.end_date
)
recipe_upstream_history_first = \
RecipeUpstreamHistory.get_first_by_date_range(
layerbranch,
milestone.start_date,
milestone.end_date,
)
if maintainer_name is None:
t_updated, t_not_updated, t_cant, t_unknown = \
Raw.get_reup_statistics(milestone.end_date, recipe_upstream_history)
milestone_statistics['all'] = \
Raw.get_reup_statistics(milestone.release.plan, milestone.end_date, recipe_upstream_history)
milestone_statistics['all'] += \
t_updated + t_not_updated + t_cant + t_unknown
milestone_statistics['up_to_date'] = t_updated
milestone_statistics['not_updated'] = t_not_updated
milestone_statistics['cant_be_updated'] = t_cant
milestone_statistics['unknown'] = t_unknown
milestone_statistics['percentage'] = 0
milestone_statistics['all_upgraded'] = 0
milestone_statistics['all_not_upgraded'] = 0
milestone_statistics['percentage_up_to_date'] = 0
milestone_statistics['percentage_not_updated'] = 0
milestone_statistics['percentage_cant_be_updated'] = 0
milestone_statistics['percentage_unknown'] = 0
milestone_statistics['up_to_date'] = +t_updated
milestone_statistics['not_updated'] = +t_not_updated
milestone_statistics['cant_be_updated'] += t_cant
milestone_statistics['unknown'] += t_unknown
if recipe_upstream_history_first:
recipes_not_upgraded = \
@ -290,11 +306,37 @@ def _get_milestone_statistics(milestone, maintainer_name=None):
recipes_upgraded = \
Raw.get_reupg_by_dates_and_recipes(
milestone.start_date, milestone.end_date, recipes_not_upgraded)
milestone_statistics['percentage'] = "%.0f" % \
((float(len(recipes_upgraded)) * 100.0)
/float(len(recipes_not_upgraded)))
milestone_statistics['all_upgraded'] = len(recipes_upgraded)
milestone_statistics['all_not_upgraded'] = len(recipes_not_upgraded)
milestone_statistics['all_upgraded'] += len(recipes_upgraded)
milestone_statistics['all_not_upgraded'] += len(recipes_not_upgraded)
else:
recipe_maintainer_history = Raw.get_remahi_by_end_date(
layerbranch.id, milestone.end_date)
recipe_maintainer_all = Raw.get_re_by_mantainer_and_date(
maintainer_name, recipe_maintainer_history[0])
milestone_statistics['all'] += len(recipe_maintainer_all)
if recipe_upstream_history:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipe_maintainer_all, recipe_upstream_history.id)
else:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipe_maintainer_all)
for ru in recipe_upstream_all:
if ru['status'] == 'Y':
milestone_statistics['up_to_date'] += 1
elif ru['status'] == 'N':
if ru['no_update_reason'] == '':
milestone_statistics['not_updated'] += 1
else:
milestone_statistics['cant_be_updated'] += 1
else:
milestone_statistics['unknown'] += 1
milestone_statistics['percentage'] = '0'
if maintainer_name is None:
if milestone_statistics['all'] > 0:
milestone_statistics['percentage_up_to_date'] = "%.0f" % \
(float(milestone_statistics['up_to_date']) * 100.0 \
/float(milestone_statistics['all']))
@ -307,37 +349,17 @@ def _get_milestone_statistics(milestone, maintainer_name=None):
milestone_statistics['percentage_unknown'] = "%.0f" % \
(float(milestone_statistics['unknown']) * 100.0
/float(milestone_statistics['all']))
if milestone_statistics['all_not_upgraded'] > 0:
milestone_statistics['percentage'] = "%.0f" % \
((float(milestone_statistics['all_upgraded']) * 100.0)
/float(milestone_statistics['all_not_upgraded']))
else:
recipe_maintainer_history = Raw.get_remahi_by_end_date(
milestone.end_date)
recipe_maintainer_all = Raw.get_re_by_mantainer_and_date(
maintainer_name, recipe_maintainer_history[0])
milestone_statistics['all'] = len(recipe_maintainer_all)
if recipe_upstream_history:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipe_maintainer_all, recipe_upstream_history.id)
else:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipe_maintainer_all)
milestone_statistics['up_to_date'] = 0
milestone_statistics['not_updated'] = 0
milestone_statistics['cant_be_updated'] = 0
milestone_statistics['unknown'] = 0
for ru in recipe_upstream_all:
if ru['status'] == 'Y':
milestone_statistics['up_to_date'] += 1
elif ru['status'] == 'N':
if ru['no_update_reason'] == '':
milestone_statistics['not_updated'] += 1
else:
milestone_statistics['cant_be_updated'] += 1
else:
milestone_statistics['unknown'] += 1
if milestone_statistics['all'] == 0:
milestone_statistics['percentage'] = '0'
milestone_statistics['percentage_up_to_date'] = "0"
milestone_statistics['percentage_not_updated'] = "0"
milestone_statistics['percentage_cant_be_updated'] = "0"
milestone_statistics['percentage_unknown'] = "0"
else:
if milestone_statistics['all'] > 0:
milestone_statistics['percentage'] = "%.0f" % \
((float(milestone_statistics['up_to_date']) /
float(milestone_statistics['all'])) * 100)
@ -361,14 +383,6 @@ class RecipeList():
self.summary = summary
def _get_recipe_list(milestone):
recipe_maintainer_history = Raw.get_remahi_by_end_date(
milestone.end_date)
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
milestone.start_date,
milestone.end_date
)
recipe_list = []
recipes_ids = []
recipe_upstream_dict_all = {}
@ -376,7 +390,19 @@ def _get_recipe_list(milestone):
maintainers_dict_all = {}
current_date = date.today()
recipes = Raw.get_reupg_by_date(milestone.end_date)
for maintplanlayer in milestone.release.plan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanlayer.layerbranch
recipe_maintainer_history = Raw.get_remahi_by_end_date(layerbranch.id,
milestone.end_date)
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
layerbranch,
milestone.start_date,
milestone.end_date
)
recipes = Raw.get_reupg_by_date(layerbranch.id, milestone.end_date)
for i,re in enumerate(recipes):
if 'pv' in re:
recipes[i]['version'] = re['pv']
@ -384,7 +410,7 @@ def _get_recipe_list(milestone):
if recipes:
recipe_last_updated = Raw.get_reup_by_last_updated(
milestone.end_date)
layerbranch.id, milestone.end_date)
for rela in recipe_last_updated:
recipe_last_updated_dict_all[rela['recipe_id']] = rela
@ -491,7 +517,10 @@ class RecipeListView(ListView):
self.milestone_statistics = _get_milestone_statistics(milestone)
self.recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date(
self.recipe_maintainer_history = {}
for maintplanlayer in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanlayer.layerbranch
self.recipe_maintainer_history[layerbranch.id] = RecipeMaintainerHistory.get_by_end_date(layerbranch,
milestone.end_date)
recipe_list = _get_recipe_list(milestone)
@ -538,8 +567,8 @@ class RecipeListView(ListView):
context['maintainer_name'] = self.maintainer_name
context['set_maintainers'] = ['All', 'No maintainer']
all_maintainers = []
for rm in RecipeMaintainer.objects.filter(history =
self.recipe_maintainer_history).values(
for layerbranch_id, rmh in self.recipe_maintainer_history.items():
for rm in RecipeMaintainer.objects.filter(history=rmh).values(
'maintainer__name').distinct().order_by('maintainer__name'):
if rm['maintainer__name'] in context['set_maintainers']:
continue
@ -608,6 +637,7 @@ def _get_recipe_upgrade_detail(maintplan, recipe_upgrade):
if milestone:
milestone_name = milestone.name
recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date(
recipe_upgrade.recipe.layerbranch,
milestone.end_date)
is_recipe_maintainer = False
@ -655,6 +685,7 @@ class RecipeDetailView(DetailView):
context['upstream_version'] = ''
context['upstream_no_update_reason'] = ''
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
recipe.layerbranch,
milestone.start_date,
milestone.end_date
)
@ -671,7 +702,7 @@ class RecipeDetailView(DetailView):
context['upstream_version'] = recipe_upstream.version
context['upstream_no_update_reason'] = recipe_upstream.no_update_reason
self.recipe_maintainer_history = RecipeMaintainerHistory.get_last()
self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(recipe.layerbranch)
recipe_maintainer = RecipeMaintainer.objects.filter(recipe = recipe,
history = self.recipe_maintainer_history)
if recipe_maintainer:
@ -737,8 +768,12 @@ class MaintainerListView(ListView):
self.milestone_statistics = _get_milestone_statistics(milestone)
self.maintainer_count = 0
for maintplanlayer in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanlayer.layerbranch
recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date(
milestone.end_date)
layerbranch, milestone.end_date)
if recipe_maintainer_history:
for rm in RecipeMaintainer.objects.filter(history =
@ -746,7 +781,7 @@ class MaintainerListView(ListView):
'maintainer__name').distinct().order_by('maintainer__name'):
maintainer_list.append(MaintainerList(rm['maintainer__name']))
self.maintainer_count = len(maintainer_list)
self.maintainer_count += len(maintainer_list)
self.intervals = sorted(intervals.keys())
current_date = date.today()