Show progress when running comparison update tasks

Provide a mechanism for distro comparison update tasks to display
progress. In practice this means the update command needs to write the
progress percentage to a file and then the log view (which is polled by
the frontend) reads this file. Originally I was going to use a FIFO for
this but that turned out to be a but unreliable; I also tried to use
Celery's state mechanism to pass it back but I simply could not get it
to work. The file-based mechanism is good enough though.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

layerindex/tools/import_otherdistro.py

@@ -359,6 +359,11 @@ def import_pkgspec(args):
         return ret
 
     updateobj = get_update_obj(args)
+    logdir = getattr(settings, 'TASK_LOG_DIR')
+    if updateobj and updateobj.task_id and logdir:
+        pwriter = utils.ProgressWriter(logdir, updateobj.task_id, logger=logger)
+    else:
+        pwriter = None
 
     metapath = args.pkgdir
 
@@ -367,7 +372,9 @@ def import_pkgspec(args):
             layerrecipes = ClassicRecipe.objects.filter(layerbranch=layerbranch)
 
             existing = list(layerrecipes.filter(deleted=False).values_list('filepath', 'filename'))
-            for entry in os.listdir(metapath):
+            dirlist = os.listdir(metapath)
+            total = len(dirlist)
+            for count, entry in enumerate(dirlist):
                 if os.path.exists(os.path.join(metapath, entry, 'dead.package')):
                     logger.info('Skipping dead package %s' % entry)
                     continue
@@ -398,6 +405,8 @@ def import_pkgspec(args):
                             rupdate.save()
                 else:
                     logger.warn('Missing spec file in %s' % os.path.join(metapath, entry))
+                if pwriter:
+                    pwriter.write(int(count / total * 100))
 
             if existing:
                 fpaths = ['%s/%s' % (pth, fn) for pth, fn in existing]

layerindex/utils.py

@@ -460,3 +460,43 @@ def timesince2(date, date2=None):
         if period:
             return '%d %s' % (period, singular if period == 1 else plural)
     return '0 seconds'
+
+class ProgressWriter():
+    def __init__(self, logdir, task_id, logger=None):
+        self.logger = logger
+        self.fn = os.path.join(logdir, '%s.progress' % task_id)
+        self.last_value = None
+        self.write('')
+
+    def write(self, value):
+        if self.fn is None:
+            return
+        if value == self.last_value:
+            return
+        try:
+            with open(self.fn + '.temp', 'w') as f:
+                f.write(str(value))
+            os.rename(self.fn + '.temp', self.fn)
+        except Exception as e:
+            if self.logger is not None:
+                self.logger.warning('Failed to write to progress file %s: %s' % (self.fn, str(e)))
+
+class ProgressReader():
+    def __init__(self, logdir, task_id, logger=None):
+        self.fn = os.path.join(logdir, '%s.progress' % task_id)
+        self.fd = None
+        self.logger = logger
+        self.last_mtime = None
+        self.last_value = None
+
+    def read(self):
+        result = None
+        try:
+            mtime = os.path.getmtime(self.fn)
+            if mtime != self.last_mtime:
+                with open(self.fn, 'r') as f:
+                    result = f.read()
+        except Exception as e:
+            if self.logger is not None:
+                self.logger.warning('Failed to read progress: %s' % str(e))
+        return result

layerindex/views.py

@@ -1400,8 +1400,11 @@ def task_log_view(request, task_id):
         response['Task-Done'] = '1'
         updateobj = get_object_or_404(Update, task_id=task_id)
         response['Task-Duration'] = utils.timesince2(updateobj.started, updateobj.finished)
+        response['Task-Progress'] = 100
     else:
         response['Task-Done'] = '0'
+        preader = utils.ProgressReader(settings.TASK_LOG_DIR, task_id)
+        response['Task-Progress'] = preader.read()
     return response
 
 class ComparisonRecipeSelectView(ClassicRecipeSearchView):

templates/layerindex/task.html

@@ -23,6 +23,13 @@
 
 <pre id="task_log" class="vertical-scroll">{{ update.log }}</pre>
 
+{% if not update.finished %}
+<div class="progress">
+    <div id="progressbar" class="progress-bar progress-bar-info progress-bar-striped" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
+    </div>
+</div>
+{% endif %}
+
 {% if update.comparisonrecipeupdate_set.exists %}
 <h3>Updated comparison recipes</h3>
 <ul>
@@ -57,6 +64,11 @@
                 posn += data.length
                 done = xhr.getResponseHeader('Task-Done')
                 duration = xhr.getResponseHeader('Task-Duration')
+                progress = parseInt(xhr.getResponseHeader('Task-Progress')) || 0;
+                if(progress) {
+                    $('#progressbar').css('width', progress + '%').attr('aria-valuenow', progress);
+                    $("#progressbar").html(progress + '%')
+                }
             }
         }).always(function () {
             if(done == '1') {