Add ability to trigger comparison updates manually from UI

Comparison updates might involve some custom fetch process, so provide a
mechanism to register these via settings.py on a per-branch basis. If
an update command is defined for a branch and the logged-in user has the
new "update_comparison_branch" permission, an "Update" button will show
up on the recipes page for the comparison branch for authenticated
users that will trigger the command in the background (as a celery job)
and then show a page that displays the status. The status isn't shown in
real-time since that requires quite a lot of plumbing, but the page at
least auto-refreshes.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2018-06-01 16:33:37 +12:00
parent ca64ab7a51
commit 9970028055
16 changed files with 310 additions and 5 deletions

View File

@ -176,6 +176,10 @@ class RecipeChangesetAdmin(admin.ModelAdmin):
RecipeChangeInline
]
class ComparisonRecipeUpdateAdmin(admin.ModelAdmin):
model = ComparisonRecipeUpdate
list_filter = ['update']
admin.site.register(Branch, BranchAdmin)
admin.site.register(YPCompatibleVersion, YPCompatibleVersionAdmin)
admin.site.register(LayerItem, LayerItemAdmin)
@ -199,5 +203,6 @@ admin.site.register(Patch)
admin.site.register(LayerRecipeExtraURL)
admin.site.register(RecipeChangeset, RecipeChangesetAdmin)
admin.site.register(ClassicRecipe, ClassicRecipeAdmin)
admin.site.register(ComparisonRecipeUpdate, ComparisonRecipeUpdateAdmin)
admin.site.register(PythonEnvironment)
admin.site.register(SiteNotice)

View File

@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.12 on 2018-06-07 04:55
from __future__ import unicode_literals
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('layerindex', '0019_layeritem_classic_comparison'),
]
operations = [
migrations.CreateModel(
name='ComparisonRecipeUpdate',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('meta_updated', models.BooleanField(default=False)),
('link_updated', models.BooleanField(default=False)),
],
),
migrations.AlterModelOptions(
name='classicrecipe',
options={'permissions': (('edit_classic', 'Can edit OE-Classic recipes'), ('update_comparison_branch', 'Can update comparison branches'))},
),
migrations.AddField(
model_name='update',
name='task_id',
field=models.CharField(blank=True, db_index=True, max_length=50),
),
migrations.AddField(
model_name='update',
name='triggered_by',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name='comparisonrecipeupdate',
name='recipe',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.ClassicRecipe'),
),
migrations.AddField(
model_name='comparisonrecipeupdate',
name='update',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.Update'),
),
]

View File

@ -95,6 +95,8 @@ class Update(models.Model):
finished = models.DateTimeField(blank=True, null=True)
log = models.TextField(blank=True)
reload = models.BooleanField('Reloaded', default=False, help_text='Was this update a reload?')
task_id = models.CharField(max_length=50, blank=True, db_index=True)
triggered_by = models.ForeignKey(User, blank=True, null=True, on_delete=models.SET_NULL)
def __str__(self):
return '%s' % self.started
@ -571,6 +573,7 @@ class ClassicRecipe(Recipe):
class Meta:
permissions = (
("edit_classic", "Can edit OE-Classic recipes"),
("update_comparison_branch", "Can update comparison branches"),
)
def get_cover_desc(self):
@ -598,6 +601,16 @@ class ClassicRecipe(Recipe):
return desc
class ComparisonRecipeUpdate(models.Model):
update = models.ForeignKey(Update)
recipe = models.ForeignKey(ClassicRecipe)
meta_updated = models.BooleanField(default=False)
link_updated = models.BooleanField(default=False)
def __str__(self):
return '%s - %s' % (self.update, self.recipe)
class Machine(models.Model):
layerbranch = models.ForeignKey(LayerBranch)
name = models.CharField(max_length=255)

View File

@ -3,6 +3,8 @@ from django.core.mail import EmailMessage
from . import utils
import os
import time
import subprocess
from datetime import datetime
try:
import settings
@ -22,3 +24,24 @@ def send_email(subject, text_content, from_email=settings.DEFAULT_FROM_EMAIL, to
utils.setup_django()
msg = EmailMessage(subject, text_content, from_email, to_emails)
msg.send()
@tasks.task(bind=True)
def run_update_command(self, branch_name, update_command):
utils.setup_django()
from layerindex.models import Update
updateobj = Update.objects.get(task_id=self.request.id)
updateobj.started = datetime.now()
updateobj.save()
output = ''
update_command = update_command.replace('%update%', str(updateobj.id))
update_command = update_command.replace('%branch%', branch_name)
try:
output = utils.runcmd(update_command, os.path.dirname(os.path.dirname(__file__)))
except subprocess.CalledProcessError as e:
output = e.output
except Exception as e:
output = str(e)
finally:
updateobj.log = output
updateobj.finished = datetime.now()
updateobj.save()

View File

@ -1,3 +1,4 @@
from datetime import datetime
from django import template
from .. import utils
@ -10,3 +11,25 @@ def replace_commas(string):
@register.filter
def squashspaces(strval):
return utils.squashspaces(strval)
@register.filter
def timesince2(date, date2=None):
# Based on http://www.didfinishlaunchingwithoptions.com/a-better-timesince-template-filter-for-django/
if date2 is None:
date2 = datetime.now()
if date > date2:
return '0 seconds'
diff = date2 - date
periods = (
(diff.days // 365, 'year', 'years'),
(diff.days // 30, 'month', 'months'),
(diff.days // 7, 'week', 'weeks'),
(diff.days, 'day', 'days'),
(diff.seconds // 3600, 'hour', 'hours'),
(diff.seconds // 60, 'minute', 'minutes'),
(diff.seconds, 'second', 'seconds'),
)
for period, singular, plural in periods:
if period:
return '%d %s' % (period, singular if period == 1 else plural)
return '0 seconds'

View File

@ -336,6 +336,17 @@ def check_branch_layer(args):
return 0, layerbranch
def get_update_obj(args):
updateobj = None
if args.update:
updateobj = Update.objects.filter(id=int(args.update))
if not updateobj:
logger.error("Specified update id %s does not exist in database" % args.update)
sys.exit(1)
updateobj = updateobj.first()
return updateobj
def import_pkgspec(args):
utils.setup_django()
import settings
@ -346,6 +357,8 @@ def import_pkgspec(args):
if ret:
return ret
updateobj = get_update_obj(args)
metapath = args.pkgdir
try:
@ -378,6 +391,10 @@ def import_pkgspec(args):
existingentry = (specpath, specfn)
if existingentry in existing:
existing.remove(existingentry)
if updateobj:
rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe)
rupdate.meta_updated = True
rupdate.save()
else:
logger.warn('Missing spec file in %s' % os.path.join(metapath, entry))
@ -452,6 +469,8 @@ def import_deblist(args):
if ret:
return ret
updateobj = get_update_obj(args)
try:
with transaction.atomic():
layerrecipes = ClassicRecipe.objects.filter(layerbranch=layerbranch)
@ -483,6 +502,10 @@ def import_deblist(args):
recipe.save()
if pkgname in existing:
existing.remove(pkgname)
if updateobj:
rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe)
rupdate.meta_updated = True
rupdate.save()
pkgs = []
pkginfo = {}
@ -545,6 +568,7 @@ def main():
parser_pkgspec.add_argument('branch', help='Branch to import into')
parser_pkgspec.add_argument('layer', help='Layer to import into')
parser_pkgspec.add_argument('pkgdir', help='Top level directory containing package subdirectories')
parser_pkgspec.add_argument('-u', '--update', help='Specify update record to link to')
parser_pkgspec.add_argument('-n', '--dry-run', help='Don\'t write any data back to the database', action='store_true')
parser_pkgspec.set_defaults(func=import_pkgspec)
@ -564,6 +588,7 @@ def main():
parser_deblist.add_argument('branch', help='Branch to import into')
parser_deblist.add_argument('layer', help='Layer to import into')
parser_deblist.add_argument('pkglistfile', help='File containing a list of packages, as produced by: apt-cache show "*"')
parser_deblist.add_argument('-u', '--update', help='Specify update record to link to')
parser_deblist.add_argument('-n', '--dry-run', help='Don\'t write any data back to the database', action='store_true')
parser_deblist.set_defaults(func=import_deblist)

View File

@ -35,6 +35,9 @@ def main():
parser.add_option("-l", "--layer",
help = "Specify layer to import into",
action="store", dest="layer", default='oe-classic')
parser.add_option("-u", "--update",
help = "Specify update record to link to",
action="store", dest="update")
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
@ -51,7 +54,7 @@ def main():
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Update, ComparisonRecipeUpdate
from django.db import transaction
logger.setLevel(options.loglevel)
@ -68,6 +71,14 @@ def main():
logger.error("Specified branch %s does not exist in database" % options.branch)
sys.exit(1)
updateobj = None
if options.update:
updateobj = Update.objects.filter(id=int(options.update))
if not updateobj:
logger.error("Specified update id %s does not exist in database" % options.update)
sys.exit(1)
updateobj = updateobj.first()
if options.skip:
skiplist = options.skip.split(',')
else:
@ -82,6 +93,7 @@ def main():
if recipe.pn in skiplist:
logger.debug('Skipping %s' % recipe.pn)
continue
updated = False
sanepn = recipe.pn.lower().replace('_', '-')
replquery = recipe_pn_query(sanepn)
found = False
@ -92,6 +104,7 @@ def main():
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
if not found:
@ -107,6 +120,7 @@ def main():
recipe.cover_status = 'P'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
if not found and recipe.pn.endswith('-nativesdk'):
@ -119,6 +133,7 @@ def main():
recipe.cover_status = 'R'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
else:
@ -137,6 +152,7 @@ def main():
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
if found:
@ -144,6 +160,7 @@ def main():
if not found:
recipe.classic_category = 'python'
recipe.save()
updated = True
elif 'cpan.org' in source0.url:
perlpn = sanepn
if perlpn.startswith('perl-'):
@ -158,18 +175,22 @@ def main():
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
if not found:
recipe.classic_category = 'perl'
recipe.save()
updated = True
if not found:
if recipe.pn.startswith('R-'):
recipe.classic_category = 'R'
recipe.save()
updated = True
elif recipe.pn.startswith('rubygem-'):
recipe.classic_category = 'ruby'
recipe.save()
updated = True
elif recipe.pn.startswith('jdk-'):
sanepn = sanepn[4:]
replquery = recipe_pn_query(sanepn)
@ -180,10 +201,12 @@ def main():
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
recipe.classic_category = 'java'
recipe.save()
updated = True
elif recipe.pn.startswith('golang-'):
if recipe.pn.startswith('golang-github-'):
sanepn = 'go-' + sanepn[14:]
@ -197,14 +220,20 @@ def main():
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
recipe.classic_category = 'go'
recipe.save()
updated = True
elif recipe.pn.startswith('gnome-'):
recipe.classic_category = 'gnome'
recipe.save()
updated = True
if updated and updateobj:
rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe)
rupdate.link_updated = True
rupdate.save()
if options.dryrun:
raise DryRunRollbackException()

View File

@ -8,7 +8,7 @@ from django.conf.urls import *
from django.views.generic import TemplateView, DetailView, ListView, RedirectView
from django.views.defaults import page_not_found
from django.core.urlresolvers import reverse_lazy
from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView
from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView, TaskStatusView
from layerindex.models import LayerItem, Recipe, RecipeChangeset
from rest_framework import routers
from . import restviews
@ -156,6 +156,10 @@ urlpatterns = [
ClassicRecipeDetailView.as_view(
template_name='layerindex/classicrecipedetail.html'),
name='comparison_recipe'),
url(r'^task/(?P<task_id>[-\w]+)/$',
TaskStatusView.as_view(
template_name='layerindex/task.html'),
name='task_status'),
url(r'^ajax/layerchecklist/(?P<branch>[-\w]+)/$',
LayerCheckListView.as_view(
template_name='layerindex/layerchecklist.html'),

View File

@ -7,7 +7,7 @@
from django.conf.urls import *
from django.views.defaults import page_not_found
from django.core.urlresolvers import reverse_lazy
from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view
from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view, comparison_update_view
urlpatterns = [
url(r'^$',
@ -44,4 +44,7 @@ urlpatterns = [
DuplicatesView.as_view(
template_name='layerindex/duplicates.html'),
name='duplicates'),
url(r'^comparison_update/$',
comparison_update_view,
name='comparison_update'),
]

View File

@ -1182,6 +1182,14 @@ class ClassicRecipeSearchView(RecipeSearchView):
else:
context['excludeclasses_display'] = ' (none)'
context['excludeclasses'] = []
context['updateable'] = False
if self.request.user.has_perm('layerindex.update_comparison_branch'):
for item in getattr(settings, 'COMPARISON_UPDATE', []):
if item['branch_name'] == context['branch'].name:
context['updateable'] = True
break
return context
@ -1327,3 +1335,43 @@ def layer_export_recipes_csv_view(request, branch, slug):
writer.writerow(values)
return response
def comparison_update_view(request, branch):
branchobj = get_object_or_404(Branch, name=branch)
if not branchobj.comparison:
raise Http404
if not request.user.has_perm('layerindex.update_comparison_branch'):
raise PermissionDenied
from celery import uuid
cmd = None
for item in getattr(settings, 'COMPARISON_UPDATE', []):
if item['branch_name'] == branchobj.name:
cmd = item['update_command']
break
if not cmd:
raise Exception('No update command defined for branch %s' % branch)
task_id = uuid()
# Create this here first, because inside the task we don't have all of the required info
update = Update(task_id=task_id)
update.started = datetime.now()
update.triggered_by = request.user
update.save()
res = tasks.run_update_command.apply_async((branch, cmd), task_id=task_id)
return HttpResponseRedirect(reverse_lazy('task_status', kwargs={'task_id': task_id}))
class TaskStatusView(TemplateView):
def get_context_data(self, **kwargs):
from celery.result import AsyncResult
context = super(TaskStatusView, self).get_context_data(**kwargs)
task_id = self.kwargs['task_id']
context['task_id'] = task_id
context['result'] = AsyncResult(task_id)
context['update'] = get_object_or_404(Update, task_id=task_id)
return context

View File

@ -20,6 +20,8 @@
<link rel="stylesheet" href="{% static "css/additional.css" %}" />
<link rel="icon" type="image/vnd.microsoft.icon" href="{% static "img/favicon.ico" %}" />
<title>{{ site_name }}{% block title_append %} - {% endblock %}</title>
{% block head_extra %}
{% endblock %}
</head>
<body>

View File

@ -50,6 +50,7 @@
<ul class="nav">
{% block navs %}{% endblock %}
</ul>
{% block navs_extra %}{% endblock %}
</div>
</div>

View File

@ -22,6 +22,28 @@
{% endautoescape %}
{% endblock %}
{% block navs_extra %}
{% autoescape on %}
{% if updateable %}
<div id="updateDialog" class="modal hide fade" tabindex="-1" role="dialog" aria-labelledby="updateDialogLabel" aria-hidden="true">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>
<h3 id="updateDialogLabel">Update {{ branch.short_description }}</h3>
</div>
<div class="modal-body">
<p>Are you sure you want to update? This will likely take some time.</p>
</div>
<div class="modal-footer">
<a href="{% url 'comparison_update' branch.name %}" class="btn btn-primary" aria-hidden="true">Update</a>
<button class="btn" id="id_layerdialog_cancel" data-dismiss="modal" aria-hidden="true">Cancel</button>
</div>
</div>
<div class="pull-right"><a href="#updateDialog" id="id_update_btn" role="button" class="btn" data-toggle="modal">Update</a></div>
{% endif %}
{% endautoescape %}
{% endblock %}
{% block content_inner %}
{% autoescape on %}

View File

@ -0,0 +1,47 @@
{% extends "base.html" %}
{% load i18n %}
{% load extrafilters %}
{% comment %}
layerindex-web - task page
Copyright (C) 2018 Intel Corporation
Licensed under the MIT license, see COPYING.MIT for details
{% endcomment %}
{% block head_extra %}
{% if not update.finished %}<meta http-equiv="refresh" content="5" />{% endif %}
{% endblock %}
<!--
{% block title_append %} - task status{% endblock %}
-->
{% block content %}
{% autoescape on %}
<p>Task status for {{ update.task_id }} started by {{ update.triggered_by }} {% if update.finished %} on {{ update.started }} (finished in {{ update.started | timesince2:update.finished }}){% else %}{{ update.started | timesince2 }} ago{% endif%}:</p>
{% if update.log %}
<pre>{{ update.log }}</pre>
{% else %}
<p>{% if update.finished %}(no output){% else %}(no output - waiting for task to finish){% endif %}
{% endif %}
{% if update.comparisonrecipeupdate_set.exists %}
<h3>Updated comparison recipes</h3>
<ul>
{% for recipeupdate in update.comparisonrecipeupdate_set.all %}
<li><a href="{% url 'comparison_recipe' recipeupdate.recipe.id %}">{{ recipeupdate.recipe.pn }}</a> {% if recipeupdate.meta_updated and recipeupdate.link_updated %}(meta, link){% elif recipeupdate.link_updated %}(link){% elif recipeupdate.meta_updated %}(meta){% endif %}</li>
{% endfor %}
</ul>
{% endif %}
{% endautoescape %}
{% endblock %}
{% block footer %}
{% endblock %}

View File

@ -38,6 +38,15 @@
<p>No messages</p>
{% endif %}
{% if update.comparisonrecipeupdate_set.exists %}
<h3>Updated comparison recipes</h3>
<ul>
{% for recipeupdate in update.comparisonrecipeupdate_set.all %}
<li><a href="{% url 'comparison_recipe' recipeupdate.recipe.id %}">{{ recipeupdate.recipe.pn }}</a> {% if recipeupdate.meta_updated and recipeupdate.link_updated %}(meta, link){% elif recipeupdate.link_updated %}(link){% elif recipeupdate.meta_updated %}(meta){% endif %}</li>
{% endfor %}
</ul>
{% endif %}
{% endautoescape %}
{% endblock %}

View File

@ -1,6 +1,7 @@
{% extends "base.html" %}
{% load i18n %}
{% load static %}
{% load extrafilters %}
{% comment %}
@ -36,7 +37,7 @@
{% for update in updates %}
<tr>
<td><a href="{% url 'update' update.id %}">{{ update.started }}{% if update.reload %} (reload){% endif %}</a></td>
<td>{% if update.finished %}{{ update.started|timesince:update.finished }}{% else %}(in progress){% endif %}</td>
<td>{% if update.finished %}{{ update.started|timesince2:update.finished }}{% else %}(in progress){% endif %}</td>
<td>{% if update.errors %}<span class="badge badge-important">{{ update.errors }}</span>{% endif %}</td>
<td>{% if update.warnings %}<span class="badge badge-warning">{{ update.warnings }}</span>{% endif %}</td>
</tr>