diff --git a/layerindex/admin.py b/layerindex/admin.py index 0fd0af3..a7c6278 100644 --- a/layerindex/admin.py +++ b/layerindex/admin.py @@ -176,6 +176,10 @@ class RecipeChangesetAdmin(admin.ModelAdmin): RecipeChangeInline ] +class ComparisonRecipeUpdateAdmin(admin.ModelAdmin): + model = ComparisonRecipeUpdate + list_filter = ['update'] + admin.site.register(Branch, BranchAdmin) admin.site.register(YPCompatibleVersion, YPCompatibleVersionAdmin) admin.site.register(LayerItem, LayerItemAdmin) @@ -199,5 +203,6 @@ admin.site.register(Patch) admin.site.register(LayerRecipeExtraURL) admin.site.register(RecipeChangeset, RecipeChangesetAdmin) admin.site.register(ClassicRecipe, ClassicRecipeAdmin) +admin.site.register(ComparisonRecipeUpdate, ComparisonRecipeUpdateAdmin) admin.site.register(PythonEnvironment) admin.site.register(SiteNotice) diff --git a/layerindex/migrations/0020_update_manual.py b/layerindex/migrations/0020_update_manual.py new file mode 100644 index 0000000..1423a82 --- /dev/null +++ b/layerindex/migrations/0020_update_manual.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.12 on 2018-06-07 04:55 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('layerindex', '0019_layeritem_classic_comparison'), + ] + + operations = [ + migrations.CreateModel( + name='ComparisonRecipeUpdate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('meta_updated', models.BooleanField(default=False)), + ('link_updated', models.BooleanField(default=False)), + ], + ), + migrations.AlterModelOptions( + name='classicrecipe', + options={'permissions': (('edit_classic', 'Can edit OE-Classic recipes'), ('update_comparison_branch', 'Can update comparison branches'))}, + ), + migrations.AddField( + model_name='update', + name='task_id', + field=models.CharField(blank=True, db_index=True, max_length=50), + ), + migrations.AddField( + model_name='update', + name='triggered_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='comparisonrecipeupdate', + name='recipe', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.ClassicRecipe'), + ), + migrations.AddField( + model_name='comparisonrecipeupdate', + name='update', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.Update'), + ), + ] diff --git a/layerindex/models.py b/layerindex/models.py index e3d2776..d7e9779 100644 --- a/layerindex/models.py +++ b/layerindex/models.py @@ -95,6 +95,8 @@ class Update(models.Model): finished = models.DateTimeField(blank=True, null=True) log = models.TextField(blank=True) reload = models.BooleanField('Reloaded', default=False, help_text='Was this update a reload?') + task_id = models.CharField(max_length=50, blank=True, db_index=True) + triggered_by = models.ForeignKey(User, blank=True, null=True, on_delete=models.SET_NULL) def __str__(self): return '%s' % self.started @@ -571,6 +573,7 @@ class ClassicRecipe(Recipe): class Meta: permissions = ( ("edit_classic", "Can edit OE-Classic recipes"), + ("update_comparison_branch", "Can update comparison branches"), ) def get_cover_desc(self): @@ -598,6 +601,16 @@ class ClassicRecipe(Recipe): return desc +class ComparisonRecipeUpdate(models.Model): + update = models.ForeignKey(Update) + recipe = models.ForeignKey(ClassicRecipe) + meta_updated = models.BooleanField(default=False) + link_updated = models.BooleanField(default=False) + + def __str__(self): + return '%s - %s' % (self.update, self.recipe) + + class Machine(models.Model): layerbranch = models.ForeignKey(LayerBranch) name = models.CharField(max_length=255) diff --git a/layerindex/tasks.py b/layerindex/tasks.py index de80804..8fdb736 100644 --- a/layerindex/tasks.py +++ b/layerindex/tasks.py @@ -3,6 +3,8 @@ from django.core.mail import EmailMessage from . import utils import os import time +import subprocess +from datetime import datetime try: import settings @@ -22,3 +24,24 @@ def send_email(subject, text_content, from_email=settings.DEFAULT_FROM_EMAIL, to utils.setup_django() msg = EmailMessage(subject, text_content, from_email, to_emails) msg.send() + +@tasks.task(bind=True) +def run_update_command(self, branch_name, update_command): + utils.setup_django() + from layerindex.models import Update + updateobj = Update.objects.get(task_id=self.request.id) + updateobj.started = datetime.now() + updateobj.save() + output = '' + update_command = update_command.replace('%update%', str(updateobj.id)) + update_command = update_command.replace('%branch%', branch_name) + try: + output = utils.runcmd(update_command, os.path.dirname(os.path.dirname(__file__))) + except subprocess.CalledProcessError as e: + output = e.output + except Exception as e: + output = str(e) + finally: + updateobj.log = output + updateobj.finished = datetime.now() + updateobj.save() diff --git a/layerindex/templatetags/extrafilters.py b/layerindex/templatetags/extrafilters.py index 7fb84db..852e426 100644 --- a/layerindex/templatetags/extrafilters.py +++ b/layerindex/templatetags/extrafilters.py @@ -1,3 +1,4 @@ +from datetime import datetime from django import template from .. import utils @@ -10,3 +11,25 @@ def replace_commas(string): @register.filter def squashspaces(strval): return utils.squashspaces(strval) + +@register.filter +def timesince2(date, date2=None): + # Based on http://www.didfinishlaunchingwithoptions.com/a-better-timesince-template-filter-for-django/ + if date2 is None: + date2 = datetime.now() + if date > date2: + return '0 seconds' + diff = date2 - date + periods = ( + (diff.days // 365, 'year', 'years'), + (diff.days // 30, 'month', 'months'), + (diff.days // 7, 'week', 'weeks'), + (diff.days, 'day', 'days'), + (diff.seconds // 3600, 'hour', 'hours'), + (diff.seconds // 60, 'minute', 'minutes'), + (diff.seconds, 'second', 'seconds'), + ) + for period, singular, plural in periods: + if period: + return '%d %s' % (period, singular if period == 1 else plural) + return '0 seconds' diff --git a/layerindex/tools/import_otherdistro.py b/layerindex/tools/import_otherdistro.py index cb8ea39..bd83438 100755 --- a/layerindex/tools/import_otherdistro.py +++ b/layerindex/tools/import_otherdistro.py @@ -336,6 +336,17 @@ def check_branch_layer(args): return 0, layerbranch +def get_update_obj(args): + updateobj = None + if args.update: + updateobj = Update.objects.filter(id=int(args.update)) + if not updateobj: + logger.error("Specified update id %s does not exist in database" % args.update) + sys.exit(1) + updateobj = updateobj.first() + return updateobj + + def import_pkgspec(args): utils.setup_django() import settings @@ -346,6 +357,8 @@ def import_pkgspec(args): if ret: return ret + updateobj = get_update_obj(args) + metapath = args.pkgdir try: @@ -378,6 +391,10 @@ def import_pkgspec(args): existingentry = (specpath, specfn) if existingentry in existing: existing.remove(existingentry) + if updateobj: + rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe) + rupdate.meta_updated = True + rupdate.save() else: logger.warn('Missing spec file in %s' % os.path.join(metapath, entry)) @@ -452,6 +469,8 @@ def import_deblist(args): if ret: return ret + updateobj = get_update_obj(args) + try: with transaction.atomic(): layerrecipes = ClassicRecipe.objects.filter(layerbranch=layerbranch) @@ -483,6 +502,10 @@ def import_deblist(args): recipe.save() if pkgname in existing: existing.remove(pkgname) + if updateobj: + rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe) + rupdate.meta_updated = True + rupdate.save() pkgs = [] pkginfo = {} @@ -545,6 +568,7 @@ def main(): parser_pkgspec.add_argument('branch', help='Branch to import into') parser_pkgspec.add_argument('layer', help='Layer to import into') parser_pkgspec.add_argument('pkgdir', help='Top level directory containing package subdirectories') + parser_pkgspec.add_argument('-u', '--update', help='Specify update record to link to') parser_pkgspec.add_argument('-n', '--dry-run', help='Don\'t write any data back to the database', action='store_true') parser_pkgspec.set_defaults(func=import_pkgspec) @@ -564,6 +588,7 @@ def main(): parser_deblist.add_argument('branch', help='Branch to import into') parser_deblist.add_argument('layer', help='Layer to import into') parser_deblist.add_argument('pkglistfile', help='File containing a list of packages, as produced by: apt-cache show "*"') + parser_deblist.add_argument('-u', '--update', help='Specify update record to link to') parser_deblist.add_argument('-n', '--dry-run', help='Don\'t write any data back to the database', action='store_true') parser_deblist.set_defaults(func=import_deblist) diff --git a/layerindex/tools/update_classic_status.py b/layerindex/tools/update_classic_status.py index 3f17923..d8cab48 100755 --- a/layerindex/tools/update_classic_status.py +++ b/layerindex/tools/update_classic_status.py @@ -35,6 +35,9 @@ def main(): parser.add_option("-l", "--layer", help = "Specify layer to import into", action="store", dest="layer", default='oe-classic') + parser.add_option("-u", "--update", + help = "Specify update record to link to", + action="store", dest="update") parser.add_option("-n", "--dry-run", help = "Don't write any data back to the database", action="store_true", dest="dryrun") @@ -51,7 +54,7 @@ def main(): options, args = parser.parse_args(sys.argv) utils.setup_django() - from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe + from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Update, ComparisonRecipeUpdate from django.db import transaction logger.setLevel(options.loglevel) @@ -68,6 +71,14 @@ def main(): logger.error("Specified branch %s does not exist in database" % options.branch) sys.exit(1) + updateobj = None + if options.update: + updateobj = Update.objects.filter(id=int(options.update)) + if not updateobj: + logger.error("Specified update id %s does not exist in database" % options.update) + sys.exit(1) + updateobj = updateobj.first() + if options.skip: skiplist = options.skip.split(',') else: @@ -82,6 +93,7 @@ def main(): if recipe.pn in skiplist: logger.debug('Skipping %s' % recipe.pn) continue + updated = False sanepn = recipe.pn.lower().replace('_', '-') replquery = recipe_pn_query(sanepn) found = False @@ -92,6 +104,7 @@ def main(): recipe.cover_status = 'D' recipe.cover_verified = False recipe.save() + updated = True found = True break if not found: @@ -107,6 +120,7 @@ def main(): recipe.cover_status = 'P' recipe.cover_verified = False recipe.save() + updated = True found = True break if not found and recipe.pn.endswith('-nativesdk'): @@ -119,6 +133,7 @@ def main(): recipe.cover_status = 'R' recipe.cover_verified = False recipe.save() + updated = True found = True break else: @@ -137,6 +152,7 @@ def main(): recipe.cover_status = 'D' recipe.cover_verified = False recipe.save() + updated = True found = True break if found: @@ -144,6 +160,7 @@ def main(): if not found: recipe.classic_category = 'python' recipe.save() + updated = True elif 'cpan.org' in source0.url: perlpn = sanepn if perlpn.startswith('perl-'): @@ -158,18 +175,22 @@ def main(): recipe.cover_status = 'D' recipe.cover_verified = False recipe.save() + updated = True found = True break if not found: recipe.classic_category = 'perl' recipe.save() + updated = True if not found: if recipe.pn.startswith('R-'): recipe.classic_category = 'R' recipe.save() + updated = True elif recipe.pn.startswith('rubygem-'): recipe.classic_category = 'ruby' recipe.save() + updated = True elif recipe.pn.startswith('jdk-'): sanepn = sanepn[4:] replquery = recipe_pn_query(sanepn) @@ -180,10 +201,12 @@ def main(): recipe.cover_status = 'D' recipe.cover_verified = False recipe.save() + updated = True found = True break recipe.classic_category = 'java' recipe.save() + updated = True elif recipe.pn.startswith('golang-'): if recipe.pn.startswith('golang-github-'): sanepn = 'go-' + sanepn[14:] @@ -197,14 +220,20 @@ def main(): recipe.cover_status = 'D' recipe.cover_verified = False recipe.save() + updated = True found = True break recipe.classic_category = 'go' recipe.save() + updated = True elif recipe.pn.startswith('gnome-'): recipe.classic_category = 'gnome' recipe.save() - + updated = True + if updated and updateobj: + rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe) + rupdate.link_updated = True + rupdate.save() if options.dryrun: raise DryRunRollbackException() diff --git a/layerindex/urls.py b/layerindex/urls.py index c8d75a2..4a78d36 100644 --- a/layerindex/urls.py +++ b/layerindex/urls.py @@ -8,7 +8,7 @@ from django.conf.urls import * from django.views.generic import TemplateView, DetailView, ListView, RedirectView from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView +from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView, TaskStatusView from layerindex.models import LayerItem, Recipe, RecipeChangeset from rest_framework import routers from . import restviews @@ -156,6 +156,10 @@ urlpatterns = [ ClassicRecipeDetailView.as_view( template_name='layerindex/classicrecipedetail.html'), name='comparison_recipe'), + url(r'^task/(?P[-\w]+)/$', + TaskStatusView.as_view( + template_name='layerindex/task.html'), + name='task_status'), url(r'^ajax/layerchecklist/(?P[-\w]+)/$', LayerCheckListView.as_view( template_name='layerindex/layerchecklist.html'), diff --git a/layerindex/urls_branch.py b/layerindex/urls_branch.py index 2809147..2fa925a 100644 --- a/layerindex/urls_branch.py +++ b/layerindex/urls_branch.py @@ -7,7 +7,7 @@ from django.conf.urls import * from django.views.defaults import page_not_found from django.core.urlresolvers import reverse_lazy -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view +from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, ClassSearchView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView, layer_export_recipes_csv_view, comparison_update_view urlpatterns = [ url(r'^$', @@ -44,4 +44,7 @@ urlpatterns = [ DuplicatesView.as_view( template_name='layerindex/duplicates.html'), name='duplicates'), + url(r'^comparison_update/$', + comparison_update_view, + name='comparison_update'), ] diff --git a/layerindex/views.py b/layerindex/views.py index 1266e3b..4993806 100644 --- a/layerindex/views.py +++ b/layerindex/views.py @@ -1182,6 +1182,14 @@ class ClassicRecipeSearchView(RecipeSearchView): else: context['excludeclasses_display'] = ' (none)' context['excludeclasses'] = [] + + context['updateable'] = False + if self.request.user.has_perm('layerindex.update_comparison_branch'): + for item in getattr(settings, 'COMPARISON_UPDATE', []): + if item['branch_name'] == context['branch'].name: + context['updateable'] = True + break + return context @@ -1327,3 +1335,43 @@ def layer_export_recipes_csv_view(request, branch, slug): writer.writerow(values) return response + + +def comparison_update_view(request, branch): + branchobj = get_object_or_404(Branch, name=branch) + if not branchobj.comparison: + raise Http404 + if not request.user.has_perm('layerindex.update_comparison_branch'): + raise PermissionDenied + + from celery import uuid + + cmd = None + for item in getattr(settings, 'COMPARISON_UPDATE', []): + if item['branch_name'] == branchobj.name: + cmd = item['update_command'] + break + if not cmd: + raise Exception('No update command defined for branch %s' % branch) + + task_id = uuid() + # Create this here first, because inside the task we don't have all of the required info + update = Update(task_id=task_id) + update.started = datetime.now() + update.triggered_by = request.user + update.save() + + res = tasks.run_update_command.apply_async((branch, cmd), task_id=task_id) + + return HttpResponseRedirect(reverse_lazy('task_status', kwargs={'task_id': task_id})) + + +class TaskStatusView(TemplateView): + def get_context_data(self, **kwargs): + from celery.result import AsyncResult + context = super(TaskStatusView, self).get_context_data(**kwargs) + task_id = self.kwargs['task_id'] + context['task_id'] = task_id + context['result'] = AsyncResult(task_id) + context['update'] = get_object_or_404(Update, task_id=task_id) + return context diff --git a/templates/base.html b/templates/base.html index a5e97cd..38cc5ea 100644 --- a/templates/base.html +++ b/templates/base.html @@ -20,6 +20,8 @@ {{ site_name }}{% block title_append %} - {% endblock %} + {% block head_extra %} + {% endblock %} diff --git a/templates/base_toplevel.html b/templates/base_toplevel.html index 7aa7f6b..1dcac77 100644 --- a/templates/base_toplevel.html +++ b/templates/base_toplevel.html @@ -50,6 +50,7 @@ + {% block navs_extra %}{% endblock %} diff --git a/templates/layerindex/classicrecipes.html b/templates/layerindex/classicrecipes.html index e796dd7..9f19d2a 100644 --- a/templates/layerindex/classicrecipes.html +++ b/templates/layerindex/classicrecipes.html @@ -22,6 +22,28 @@ {% endautoescape %} {% endblock %} +{% block navs_extra %} +{% autoescape on %} +{% if updateable %} + + +{% endif %} +{% endautoescape %} +{% endblock %} + + {% block content_inner %} {% autoescape on %} diff --git a/templates/layerindex/task.html b/templates/layerindex/task.html new file mode 100644 index 0000000..371945e --- /dev/null +++ b/templates/layerindex/task.html @@ -0,0 +1,47 @@ +{% extends "base.html" %} +{% load i18n %} +{% load extrafilters %} + +{% comment %} + + layerindex-web - task page + + Copyright (C) 2018 Intel Corporation + Licensed under the MIT license, see COPYING.MIT for details + +{% endcomment %} + + +{% block head_extra %} +{% if not update.finished %}{% endif %} +{% endblock %} + + + +{% block content %} +{% autoescape on %} + +

Task status for {{ update.task_id }} started by {{ update.triggered_by }} {% if update.finished %} on {{ update.started }} (finished in {{ update.started | timesince2:update.finished }}){% else %}{{ update.started | timesince2 }} ago{% endif%}:

+ +{% if update.log %} +
{{ update.log }}
+{% else %} +

{% if update.finished %}(no output){% else %}(no output - waiting for task to finish){% endif %} +{% endif %} + +{% if update.comparisonrecipeupdate_set.exists %} +

Updated comparison recipes

+ +{% endif %} + +{% endautoescape %} +{% endblock %} + +{% block footer %} +{% endblock %} diff --git a/templates/layerindex/updatedetail.html b/templates/layerindex/updatedetail.html index 05e115f..59723fe 100644 --- a/templates/layerindex/updatedetail.html +++ b/templates/layerindex/updatedetail.html @@ -38,6 +38,15 @@

No messages

{% endif %} +{% if update.comparisonrecipeupdate_set.exists %} +

Updated comparison recipes

+ +{% endif %} + {% endautoescape %} {% endblock %} diff --git a/templates/layerindex/updatelist.html b/templates/layerindex/updatelist.html index 0549dba..d58d175 100644 --- a/templates/layerindex/updatelist.html +++ b/templates/layerindex/updatelist.html @@ -1,6 +1,7 @@ {% extends "base.html" %} {% load i18n %} {% load static %} +{% load extrafilters %} {% comment %} @@ -36,7 +37,7 @@ {% for update in updates %} {{ update.started }}{% if update.reload %} (reload){% endif %} - {% if update.finished %}{{ update.started|timesince:update.finished }}{% else %}(in progress){% endif %} + {% if update.finished %}{{ update.started|timesince2:update.finished }}{% else %}(in progress){% endif %} {% if update.errors %}{{ update.errors }}{% endif %} {% if update.warnings %}{{ update.warnings }}{% endif %}