Record and display update logs

At the moment it's a bit difficult to get update logs out of the
environment in which the update script is being run. In order to make
the logs more accessible, create a LayerUpdate model to record the
output of update_layer.py separately for each layerbranch and tie the
created LayerUpdates together with a single Update model per session.

We provide two ways to look at this - a Tools->Updates page for
logged-in users, and there's also an "Updates" tab on each layer that is
accessible to anyone; which one is useful depends on whether you are
looking at the index as a whole or an individual layer.

Update records older than 30 days are deleted automatically by default.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

TODO

@@ -35,10 +35,8 @@ Other
 * Create simple script to check for unlisted layer subdirectories in all repos
 * Auto-detect more values from github pages?
 * Ability for submitters to get email notification about publication?
-* Update script still seems not to be always printing layer name on parsing warnings/errors
 * Update script could send warnings when parsing layers to maintainers? (optional)
 * Click on OE-Classic graph element to go to query?
 * Use bar instead of pie graphs for OE-Classic statistics
 * Ensure OE-Core appears before meta-oe in layer list
 * Ability for reviewers to comment before publishing a layer?
-* Record update & parse errors against recipe/layer

layerindex/admin.py

@@ -74,6 +74,12 @@ class LayerDependencyAdmin(CompareVersionAdmin):
 class LayerNoteAdmin(CompareVersionAdmin):
     list_filter = ['layer__name']
 
+class UpdateAdmin(admin.ModelAdmin):
+    pass
+
+class LayerUpdateAdmin(admin.ModelAdmin):
+    list_filter = ['update__started', 'layerbranch__layer__name', 'layerbranch__branch__name']
+
 class RecipeAdmin(admin.ModelAdmin):
     search_fields = ['filename', 'pn']
     list_filter = ['layerbranch__layer__name', 'layerbranch__branch__name']
@@ -144,6 +150,8 @@ admin.site.register(LayerBranch, LayerBranchAdmin)
 admin.site.register(LayerMaintainer, LayerMaintainerAdmin)
 admin.site.register(LayerDependency, LayerDependencyAdmin)
 admin.site.register(LayerNote, LayerNoteAdmin)
+admin.site.register(Update, UpdateAdmin)
+admin.site.register(LayerUpdate, LayerUpdateAdmin)
 admin.site.register(Recipe, RecipeAdmin)
 admin.site.register(RecipeFileDependency)
 admin.site.register(Machine, MachineAdmin)

layerindex/migrations/0005_layerupdate.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layerindex', '0004_layerdependency_required'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LayerUpdate',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('started', models.DateTimeField()),
+                ('finished', models.DateTimeField()),
+                ('errors', models.IntegerField(default=0)),
+                ('warnings', models.IntegerField(default=0)),
+                ('log', models.TextField(blank=True)),
+                ('layerbranch', models.ForeignKey(to='layerindex.LayerBranch')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Update',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('started', models.DateTimeField()),
+                ('finished', models.DateTimeField(null=True, blank=True)),
+                ('log', models.TextField(blank=True)),
+                ('reload', models.BooleanField(help_text='Was this update a reload?', verbose_name='Reloaded', default=False)),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='branch',
+            name='name',
+            field=models.CharField(max_length=50, verbose_name='Branch name'),
+        ),
+        migrations.AddField(
+            model_name='layerupdate',
+            name='update',
+            field=models.ForeignKey(to='layerindex.Update'),
+        ),
+    ]

layerindex/models.py

@@ -31,7 +31,7 @@ class PythonEnvironment(models.Model):
 
 
 class Branch(models.Model):
-    name = models.CharField(max_length=50)
+    name = models.CharField('Branch name', max_length=50)
     bitbake_branch = models.CharField(max_length=50)
     short_description = models.CharField(max_length=50, blank=True)
     sort_priority = models.IntegerField(blank=True, null=True)
@@ -47,6 +47,16 @@ class Branch(models.Model):
         return self.name
 
 
+class Update(models.Model):
+    started = models.DateTimeField()
+    finished = models.DateTimeField(blank=True, null=True)
+    log = models.TextField(blank=True)
+    reload = models.BooleanField('Reloaded', default=False, help_text='Was this update a reload?')
+
+    def __str__(self):
+        return '%s' % self.started
+
+
 class LayerItem(models.Model):
     LAYER_STATUS_CHOICES = (
         ('N', 'New'),
@@ -255,6 +265,31 @@ class LayerNote(models.Model):
         return "%s: %s" % (self.layer.name, self.text)
 
 
+class LayerUpdate(models.Model):
+    layerbranch = models.ForeignKey(LayerBranch)
+    update = models.ForeignKey(Update)
+    started = models.DateTimeField()
+    finished = models.DateTimeField()
+    errors = models.IntegerField(default=0)
+    warnings = models.IntegerField(default=0)
+    log = models.TextField(blank=True)
+
+    def save(self):
+        warnings = 0
+        errors = 0
+        for line in self.log.splitlines():
+            if line.startswith('WARNING:'):
+                warnings += 1
+            elif line.startswith('ERROR:'):
+                errors += 1
+        self.warnings = warnings
+        self.errors = errors
+        super(LayerUpdate, self).save()
+
+    def __str__(self):
+        return "%s: %s: %s" % (self.layerbranch.layer.name, self.layerbranch.branch.name, self.started)
+
+
 class Recipe(models.Model):
     layerbranch = models.ForeignKey(LayerBranch)
     filename = models.CharField(max_length=255)

layerindex/update.py

@@ -14,6 +14,7 @@ import optparse
 import logging
 import subprocess
 import signal
+from datetime import datetime, timedelta
 from distutils.version import LooseVersion
 import utils
 from layerconfparse import LayerConfParse
@@ -41,10 +42,24 @@ def run_command_interruptible(cmd):
     """
     signal.signal(signal.SIGINT, signal.SIG_IGN)
     try:
-        ret = subprocess.call(cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint)
+        process = subprocess.Popen(
+            cmd, cwd=os.path.dirname(sys.argv[0]), shell=True, preexec_fn=reenable_sigint, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        )
+
+        buf = ''
+        while True:
+            out = process.stdout.read(1)
+            out = out.decode('utf-8')
+            if out:
+                sys.stdout.write(out)
+                sys.stdout.flush()
+                buf += out
+            elif out == '' and process.poll() != None:
+                break
+
     finally:
         signal.signal(signal.SIGINT, signal.SIG_DFL)
-    return ret
+    return process.returncode, buf
 
 
 def main():
@@ -93,7 +108,7 @@ def main():
 
     utils.setup_django()
     import settings
-    from layerindex.models import Branch, LayerItem, LayerDependency
+    from layerindex.models import Branch, LayerItem, Update, LayerUpdate
 
     logger.setLevel(options.loglevel)
 
@@ -126,130 +141,171 @@ def main():
     if not os.path.exists(fetchdir):
         os.makedirs(fetchdir)
     fetchedrepos = []
-    failedrepos = []
+    failedrepos = {}
 
-    lockfn = os.path.join(fetchdir, "layerindex.lock")
-    lockfile = utils.lock_file(lockfn)
-    if not lockfile:
-        logger.error("Layer index lock timeout expired")
-        sys.exit(1)
+    listhandler = utils.ListHandler()
+    listhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
+    logger.addHandler(listhandler)
+
+    update = Update()
+    update.started = datetime.now()
+    if options.fullreload or options.reload:
+        update.reload = True
+    else:
+        update.reload = False
+    if not options.dryrun:
+        update.save()
     try:
-        bitbakepath = os.path.join(fetchdir, 'bitbake')
+        lockfn = os.path.join(fetchdir, "layerindex.lock")
+        lockfile = utils.lock_file(lockfn)
+        if not lockfile:
+            logger.error("Layer index lock timeout expired")
+            sys.exit(1)
+        try:
+            bitbakepath = os.path.join(fetchdir, 'bitbake')
 
-        if not options.nofetch:
-            # Fetch latest metadata from repositories
-            for layer in layerquery:
-                # Handle multiple layers in a single repo
-                urldir = layer.get_fetch_dir()
-                repodir = os.path.join(fetchdir, urldir)
-                if not (layer.vcs_url in fetchedrepos or layer.vcs_url in failedrepos):
-                    logger.info("Fetching remote repository %s" % layer.vcs_url)
-                    out = None
-                    try:
-                        if not os.path.exists(repodir):
-                            out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
-                        else:
-                            out = utils.runcmd("git fetch", repodir, logger=logger)
-                    except Exception as e:
-                        logger.error("Fetch of layer %s failed: %s" % (layer.name, str(e)))
-                        failedrepos.append(layer.vcs_url)
-                        continue
-                    fetchedrepos.append(layer.vcs_url)
-
-            if not fetchedrepos:
-                logger.error("No repositories could be fetched, exiting")
-                sys.exit(1)
-
-            logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL)
-            if not os.path.exists(bitbakepath):
-                out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger)
-            else:
-                out = utils.runcmd("git fetch", bitbakepath, logger=logger)
-
-        # Process and extract data from each layer
-        # We now do this by calling out to a separate script; doing otherwise turned out to be
-        # unreliable due to leaking memory (we're using bitbake internals in a manner in which
-        # they never get used during normal operation).
-        last_rev = {}
-        for branch in branches:
-            for layer in layerquery:
-                if layer.vcs_url in failedrepos:
-                    logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
-                    continue
-
-                urldir = layer.get_fetch_dir()
-                repodir = os.path.join(fetchdir, urldir)
-
-                branchobj = utils.get_branch(branch)
-
-                if branchobj.update_environment:
-                    cmdprefix = branchobj.update_environment.get_command()
-                else:
-                    cmdprefix = 'python3'
-                cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch)
-                if options.reload:
-                    cmd += ' --reload'
-                if options.fullreload:
-                    cmd += ' --fullreload'
-                if options.nocheckout:
-                    cmd += ' --nocheckout'
-                if options.dryrun:
-                    cmd += ' -n'
-                if options.loglevel == logging.DEBUG:
-                    cmd += ' -d'
-                elif options.loglevel == logging.ERROR:
-                    cmd += ' -q'
-                logger.debug('Running layer update command: %s' % cmd)
-                ret = run_command_interruptible(cmd)
-
-                # We need to get layerbranch here because it might not have existed until
-                # layer_update.py created it, but it still may not create one (e.g. if subdir
-                # didn't exist) so we still need to check
-                layerbranch = layer.get_layerbranch(branch)
-                if layerbranch:
-                    last_rev[layerbranch] = layerbranch.vcs_last_rev
-
-                if ret == 254:
-                    # Interrupted by user, break out of loop
-                    break
-
-        # Since update_layer may not be called in the correct order to have the
-        # dependencies created before trying to link them, we now have to loop
-        # back through all the branches and layers and try to link in the
-        # dependencies that may have been missed.  Note that creating the
-        # dependencies is a best-effort and continues if they are not found.
-        for branch in branches:
-            try:
-                layerconfparser = LayerConfParse(logger=logger, bitbakepath=bitbakepath)
+            if not options.nofetch:
+                # Fetch latest metadata from repositories
                 for layer in layerquery:
+                    # Handle multiple layers in a single repo
+                    urldir = layer.get_fetch_dir()
+                    repodir = os.path.join(fetchdir, urldir)
+                    if not (layer.vcs_url in fetchedrepos or layer.vcs_url in failedrepos):
+                        logger.info("Fetching remote repository %s" % layer.vcs_url)
+                        out = None
+                        try:
+                            if not os.path.exists(repodir):
+                                out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger, printerr=False)
+                            else:
+                                out = utils.runcmd("git fetch", repodir, logger=logger, printerr=False)
+                        except subprocess.CalledProcessError as e:
+                            logger.error("Fetch of layer %s failed: %s" % (layer.name, e.output))
+                            failedrepos[layer.vcs_url] = e.output
+                            continue
+                        fetchedrepos.append(layer.vcs_url)
 
-                    layerbranch = layer.get_layerbranch(branch)
-                    # Skip layers that did not change.
-                    layer_last_rev = None
-                    if layerbranch:
-                        layer_last_rev = last_rev.get(layerbranch, None)
-                    if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev:
+                if not fetchedrepos:
+                    logger.error("No repositories could be fetched, exiting")
+                    sys.exit(1)
+
+                logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL)
+                if not os.path.exists(bitbakepath):
+                    out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger)
+                else:
+                    out = utils.runcmd("git fetch", bitbakepath, logger=logger)
+
+            # Process and extract data from each layer
+            # We now do this by calling out to a separate script; doing otherwise turned out to be
+            # unreliable due to leaking memory (we're using bitbake internals in a manner in which
+            # they never get used during normal operation).
+            last_rev = {}
+            for branch in branches:
+                for layer in layerquery:
+                    layerupdate = LayerUpdate()
+                    layerupdate.update = update
+
+                    errmsg = failedrepos.get(layer.vcs_url, '')
+                    if errmsg:
+                        logger.info("Skipping update of layer %s as fetch of repository %s failed:\n%s" % (layer.name, layer.vcs_url, errmsg))
+                        layerbranch = layer.get_layerbranch(branch)
+                        if layerbranch:
+                            layerupdate.layerbranch = layerbranch
+                            layerupdate.started = datetime.now()
+                            layerupdate.finished = datetime.now()
+                            layerupdate.log = 'ERROR: fetch failed: %s' % errmsg
+                            if not options.dryrun:
+                                layerupdate.save()
                         continue
 
                     urldir = layer.get_fetch_dir()
                     repodir = os.path.join(fetchdir, urldir)
 
-                    utils.checkout_layer_branch(layerbranch, repodir, logger)
+                    branchobj = utils.get_branch(branch)
 
-                    config_data = layerconfparser.parse_layer(layerbranch, repodir)
-                    if not config_data:
-                        logger.debug("Layer %s does not appear to have branch %s" % (layer.name, branch))
-                        continue
+                    if branchobj.update_environment:
+                        cmdprefix = branchobj.update_environment.get_command()
+                    else:
+                        cmdprefix = 'python3'
+                    cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch)
+                    if options.reload:
+                        cmd += ' --reload'
+                    if options.fullreload:
+                        cmd += ' --fullreload'
+                    if options.nocheckout:
+                        cmd += ' --nocheckout'
+                    if options.dryrun:
+                        cmd += ' -n'
+                    if options.loglevel == logging.DEBUG:
+                        cmd += ' -d'
+                    elif options.loglevel == logging.ERROR:
+                        cmd += ' -q'
 
-                    utils.add_dependencies(layerbranch, config_data, logger=logger)
-                    utils.add_recommends(layerbranch, config_data, logger=logger)
-            finally:
-                layerconfparser.shutdown()
+                    logger.debug('Running layer update command: %s' % cmd)
+                    layerupdate.started = datetime.now()
+                    ret, output = run_command_interruptible(cmd)
+                    layerupdate.finished = datetime.now()
 
+                    # We need to get layerbranch here because it might not have existed until
+                    # layer_update.py created it, but it still may not create one (e.g. if subdir
+                    # didn't exist) so we still need to check
+                    layerbranch = layer.get_layerbranch(branch)
+                    if layerbranch:
+                        last_rev[layerbranch] = layerbranch.vcs_last_rev
+                        layerupdate.layerbranch = layerbranch
+                        layerupdate.log = output
+                        if not options.dryrun:
+                            layerupdate.save()
 
+                    if ret == 254:
+                        # Interrupted by user, break out of loop
+                        break
+
+            # Since update_layer may not be called in the correct order to have the
+            # dependencies created before trying to link them, we now have to loop
+            # back through all the branches and layers and try to link in the
+            # dependencies that may have been missed.  Note that creating the
+            # dependencies is a best-effort and continues if they are not found.
+            for branch in branches:
+                try:
+                    layerconfparser = LayerConfParse(logger=logger, bitbakepath=bitbakepath)
+                    for layer in layerquery:
+
+                        layerbranch = layer.get_layerbranch(branch)
+                        # Skip layers that did not change.
+                        layer_last_rev = None
+                        if layerbranch:
+                            layer_last_rev = last_rev.get(layerbranch, None)
+                        if layer_last_rev is None or layer_last_rev == layerbranch.vcs_last_rev:
+                            continue
+
+                        urldir = layer.get_fetch_dir()
+                        repodir = os.path.join(fetchdir, urldir)
+
+                        utils.checkout_layer_branch(layerbranch, repodir, logger)
+
+                        config_data = layerconfparser.parse_layer(layerbranch, repodir)
+                        if not config_data:
+                            logger.debug("Layer %s does not appear to have branch %s" % (layer.name, branch))
+                            continue
+
+                        utils.add_dependencies(layerbranch, config_data, logger=logger)
+                        utils.add_recommends(layerbranch, config_data, logger=logger)
+                finally:
+                    layerconfparser.shutdown()
+
+        finally:
+            utils.unlock_file(lockfile)
 
     finally:
-        utils.unlock_file(lockfile)
+        update.log = ''.join(listhandler.read())
+        update.finished = datetime.now()
+        if not options.dryrun:
+            update.save()
+
+    if not options.dryrun:
+        # Purge old update records
+        update_purge_days = getattr(settings, 'UPDATE_PURGE_DAYS', 30)
+        Update.objects.filter(started__lte=datetime.now()-timedelta(days=update_purge_days)).delete()
 
     sys.exit(0)
 

layerindex/urls.py

@@ -8,7 +8,7 @@ from django.conf.urls import *
 from django.views.generic import TemplateView, DetailView, ListView, RedirectView
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
-from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView
+from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, UpdateDetailView
 from layerindex.models import LayerItem, Recipe, RecipeChangeset
 from rest_framework import routers
 from . import restviews
@@ -67,6 +67,10 @@ urlpatterns = patterns('',
             template_name='layerindex/recipedetail.html'),
             name='recipe'),
     url(r'^layer/(?P<name>[-\w]+)/publish/$', 'layerindex.views.publish', name="publish"),
+    url(r'^layerupdate/(?P<pk>[-\w]+)/$',
+        LayerUpdateDetailView.as_view(
+            template_name='layerindex/layerupdate.html'),
+            name='layerupdate'),
     url(r'^bulkchange/$',
         BulkChangeView.as_view(
             template_name='layerindex/bulkchange.html'),
@@ -97,6 +101,14 @@ urlpatterns = patterns('',
     #        context_object_name='recipe_list',
     #        template_name='layerindex/rawrecipes.txt'),
     #        name='recipe_list_raw'),
+    url(r'^updates/$',
+        UpdateListView.as_view(
+            template_name='layerindex/updatelist.html'),
+            name='update_list'),
+    url(r'^updates/(?P<pk>[-\w]+)/$',
+        UpdateDetailView.as_view(
+            template_name='layerindex/updatedetail.html'),
+            name='update'),
     url(r'^history/$',
         HistoryListView.as_view(
             template_name='layerindex/history.html'),

layerindex/urls_branch.py

@@ -1,13 +1,13 @@
 # layerindex-web - Branch-based URL definitions
 #
-# Copyright (C) 2013 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
 from django.conf.urls import *
 from django.views.defaults import page_not_found
 from django.core.urlresolvers import reverse_lazy
-from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView
+from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, DistroSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, RedirectParamsView, DuplicatesView, LayerUpdateDetailView
 
 urlpatterns = patterns('',
     url(r'^$', 

layerindex/utils.py

@@ -223,6 +223,19 @@ def logger_create(name):
     logger.setLevel(logging.INFO)
     return logger
 
+class ListHandler(logging.Handler):
+    """Logging handler which accumulates formatted log records in a list, returning the list on demand"""
+    def __init__(self):
+        self.log = []
+        logging.Handler.__init__(self, logging.WARNING)
+    def emit(self, record):
+        self.log.append('%s\n' % self.format(record))
+    def read(self):
+        log = self.log
+        self.log = []
+        return log
+
+
 def lock_file(fn):
     starttime = time.time()
     while True:

layerindex/views.py

@@ -1,6 +1,6 @@
 # layerindex-web - view definitions
 #
-# Copyright (C) 2013-2014 Intel Corporation
+# Copyright (C) 2013-2016 Intel Corporation
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
@@ -10,7 +10,7 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidde
 from django.core.urlresolvers import reverse, reverse_lazy, resolve
 from django.core.exceptions import PermissionDenied
 from django.template import RequestContext
-from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Recipe, Machine, Distro, BBClass, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe
+from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Update, LayerUpdate, Recipe, Machine, Distro, BBClass, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe
 from datetime import datetime
 from django.views.generic import TemplateView, DetailView, ListView
 from django.views.generic.edit import CreateView, DeleteView, UpdateView
@@ -18,7 +18,7 @@ from django.views.generic.base import RedirectView
 from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm, EditProfileForm, RecipeChangesetForm, AdvancedRecipeSearchForm, BulkChangeEditFormSet, ClassicRecipeForm, ClassicRecipeSearchForm
 from django.db import transaction
 from django.contrib.auth.models import User, Permission
-from django.db.models import Q, Count
+from django.db.models import Q, Count, Sum
 from django.core.mail import EmailMessage
 from django.template.loader import get_template
 from django.template import Context
@@ -328,6 +328,7 @@ class LayerDetailView(DetailView):
             context['distros'] = layerbranch.distro_set.order_by('name')
             context['appends'] = layerbranch.bbappend_set.order_by('filename')
             context['classes'] = layerbranch.bbclass_set.order_by('name')
+            context['updates'] = layerbranch.layerupdate_set.order_by('-started')
         context['url_branch'] = self.kwargs['branch']
         context['this_url_name'] = resolve(self.request.path_info).url_name
         return context
@@ -599,6 +600,29 @@ class MachineSearchView(ListView):
         return context
 
 
+class UpdateListView(ListView):
+    context_object_name = "updates"
+    paginate_by = 50
+
+    def get_queryset(self):
+        return Update.objects.all().order_by('-started').annotate(errors=Sum('layerupdate__errors'), warnings=Sum('layerupdate__warnings'))
+
+
+class UpdateDetailView(DetailView):
+    model = Update
+
+    def get_context_data(self, **kwargs):
+        context = super(UpdateDetailView, self).get_context_data(**kwargs)
+        update = self.get_object()
+        if update:
+            context['layerupdates'] = update.layerupdate_set.exclude(log__isnull=True).exclude(log__exact='')
+        return context
+
+
+class LayerUpdateDetailView(DetailView):
+    model = LayerUpdate
+
+
 class DistroSearchView(ListView):
     context_object_name = 'distro_list'
     paginate_by = 50

settings.py

@@ -211,6 +211,9 @@ BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake"
 # Core layer to be used by the update script for basic BitBake configuration
 CORE_LAYER_NAME = "openembedded-core"
 
+# Update records older than this number of days will be deleted every update
+UPDATE_PURGE_DAYS = 30
+
 # Settings for layer submission feature
 SUBMIT_EMAIL_FROM = 'noreply@example.com'
 SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'

templates/base.html

@@ -72,6 +72,7 @@
                         <ul class="dropdown-menu">
                             <li><a href="{% url 'bulk_change' %}">Bulk Change</a></li>
                             <li><a href="{% url 'duplicates' 'master' %}">Duplicates</a></li>
+                            <li><a href="{% url 'update_list' %}">Updates</a></li>
                         </ul>
                     </li>
                     {% endif %}

templates/layerindex/detail.html

@@ -181,6 +181,9 @@
             {% if distros.count > 0 %}
                 <li><a href="#distros" data-toggle="tab">Distros</a></li>
             {% endif %}
+            {% if updates.count > 0 %}
+                <li><a href="#updates" data-toggle="tab">Updates</a></li>
+            {% endif %}
         </ul>
 
         <div class="tab-content">
@@ -298,6 +301,40 @@
                     </table>
                 </div>
             {% endif %}
+            {% if updates.count > 0 %}
+                <div class="tab-pane" id="updates">
+                    <div class="navbar">
+                        <div class="navbar-inner">
+                            <a class="brand pull-left">{{ layeritem.name }} updates</a>
+                        </div>
+                    </div>
+
+                    <table class="table table-bordered">
+                        <thead>
+                            <tr>
+                                <th>Date/time</th>
+                                <th>Errors</th>
+                                <th>Warnings</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+                            {% for update in updates %}
+                                <tr>
+                                    <td>
+                                        {% if update.log %}
+                                            <a href="{% url 'layerupdate' update.id %}">{{ update.started }}{% if update.update.reload %} (reload){% endif%}</a>
+                                        {% else %}
+                                            <span class="muted">{{ update.started }}{% if update.update.reload %} (reload){% endif%}</span>
+                                        {% endif %}
+                                    </td>
+                                    <td>{% if update.errors %}<span class="badge badge-important">{{ update.errors }}</span>{% endif %}</td>
+                                    <td>{% if update.warnings %}<span class="badge badge-warning">{{ update.warnings }}</span>{% endif %}</td>
+                                </tr>
+                            {% endfor %}
+                        </tbody>
+                    </table>
+                </div>
+            {% endif %}
         </div>
  
 

templates/layerindex/layerupdate.html

@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+{% load i18n %}
+
+{% comment %}
+
+  layerindex-web - layer update page
+
+  Copyright (C) 2016 Intel Corporation
+  Licensed under the MIT license, see COPYING.MIT for details
+
+{% endcomment %}
+
+<!--
+{% block title_append %} - {{ layerupdate.layerbranch.layer.name }} {{ layerupdate.layerbranch.branch.name }} - {{ layerupdate.started }} {% endblock %}
+-->
+
+{% block content %}
+{% autoescape on %}
+
+<h2>{{ layerupdate.layerbranch.layer.name }} {{ layerupdate.layerbranch.branch.name }} - {{ layerupdate.started }}</h2>
+
+<pre>{{ layerupdate.log }}</pre>
+
+{% endautoescape %}
+
+{% endblock %}
+
+
+{% block scripts %}
+{% endblock %}

templates/layerindex/updatedetail.html

@@ -0,0 +1,47 @@
+{% extends "base.html" %}
+{% load i18n %}
+
+{% comment %}
+
+  layerindex-web - update page
+
+  Copyright (C) 2016 Intel Corporation
+  Licensed under the MIT license, see COPYING.MIT for details
+
+{% endcomment %}
+
+<!--
+{% block title_append %} - {{ update.started }} {% endblock %}
+-->
+
+{% block content %}
+{% autoescape on %}
+
+        <ul class="breadcrumb">
+            <li><a href="{% url 'update_list' %}">Updates</a> <span class="divider">&rarr;</span></li>
+            <li class="active">{{ update.started }}</li>
+        </ul>
+
+
+<h2>{{ update.started }} {% if update.reload %}(reload){% endif %}</h2>
+
+{% if update.log %}
+    <pre>{{ update.log }}</pre>
+{% endif %}
+
+{% for layerupdate in layerupdates %}
+    <a href="{% url 'layer_item' layerupdate.layerbranch.branch.name layerupdate.layerbranch.layer.name %}"><h3>{{ layerupdate.layerbranch.layer.name }} {{ layerupdate.layerbranch.branch.name }}</h3></a>
+    <pre>{{ layerupdate.log }}</pre>
+{% endfor %}
+
+{% if not update.log and not layerupdates %}
+    <p>No messages</p>
+{% endif %}
+
+{% endautoescape %}
+
+{% endblock %}
+
+
+{% block scripts %}
+{% endblock %}

templates/layerindex/updatelist.html

@@ -0,0 +1,57 @@
+{% extends "base.html" %}
+{% load i18n %}
+{% load static %}
+
+{% comment %}
+
+  layerindex-web - updates list page template
+
+  Copyright (C) 2016 Intel Corporation
+  Licensed under the MIT license, see COPYING.MIT for details
+
+{% endcomment %}
+
+
+<!--
+{% block title_append %} - updates{% endblock %}
+-->
+
+{% block content %}
+{% autoescape on %}
+
+<div class="row-fluid">
+    <div class="span9 offset1">
+
+        <table class="table table-striped table-bordered">
+            <thead>
+                <tr>
+                    <th>Update date</th>
+                    <th>Time</th>
+                    <th>Errors</th>
+                    <th>Warnings</th>
+                </tr>
+            </thead>
+
+            <tbody>
+                {% for update in updates %}
+                <tr>
+                    <td><a href="{% url 'update' update.id %}">{{ update.started }}{% if update.reload %} (reload){% endif %}</a></td>
+                    <td>{% if update.finished %}{{ update.started|timesince:update.finished }}{% else %}(in progress){% endif %}</td>
+                    <td>{% if update.errors %}<span class="badge badge-important">{{ update.errors }}</span>{% endif %}</td>
+                    <td>{% if update.warnings %}<span class="badge badge-warning">{{ update.warnings }}</span>{% endif %}</td>
+                </tr>
+                {% endfor %}
+
+            </tbody>
+        </table>
+    </div>
+</div>
+
+{% if is_paginated %}
+    {% load pagination %}
+    {% pagination page_obj %}
+{% endif %}
+
+{% endautoescape %}
+
+{% endblock %}