rrs_upstream_email: rework

* Use maintenance plans to get layerbranches
* Use from/to/subject and admin contact from maintenance plan
* Use an actual template to render the email (and drop tabulate
  dependency)
* Improve grammar in the email text
* Use a single line to represent the most recent commit

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

requirements.txt

@@ -27,4 +27,3 @@ smmap==0.9.0
 smmap2==2.0.3
 Unidecode==0.4.19
 vine==1.1.4
-tabulate==0.7.3

rrs/tools/rrs_upstream_email.py

@@ -3,7 +3,7 @@
 
 # Send email to maintainers about the current status of the recipes.
 #
-# Copyright (C) 2015 Intel Corporation
+# Copyright (C) 2015, 2018 Intel Corporation
 # Author: Aníbal Limón <anibal.limon@linux.intel.com>
 #
 # Licensed under the MIT license, see COPYING.MIT for details
@@ -12,7 +12,7 @@ import sys
 import os.path
 import optparse
 import logging
-from tabulate import tabulate
+from collections import namedtuple
 
 sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__))))
 from common import common_setup, get_pv_type, get_logger
@@ -21,43 +21,30 @@ from layerindex import utils
 
 utils.setup_django()
 from django.core.mail import EmailMessage
+from django.template.loader import get_template
+from django.template import Context, Template
+from django.core.urlresolvers import reverse
+from django.contrib.sites.models import Site
 import settings
 
 from layerindex.models import LayerItem, LayerBranch, Recipe
 from rrs.models import Maintainer, RecipeMaintainerHistory, RecipeMaintainer, \
-    RecipeUpstream, RecipeUpstreamHistory
+    RecipeUpstream, RecipeUpstreamHistory, MaintenancePlan
 
 logger = get_logger('UpstreamEmail', settings)
 
-LAYERBRANCH_NAME = "master"
 
 """
     Send email with Recipes that need update.
 """
-def send_email(recipes, repodir, options):
+def send_email(maintplan, recipes, options):
-    header = """This mail was sent out by Recipe reporting system.
-
-This message list those recipes which need to be upgraded. If maintainers
-believe some of them needn't to upgrade at this time, they can fill
-RECIPE_NO_UPDATE_REASON in respective recipe file to ignore this remainder
-until newer upstream version was detected.
-
-Example:
-RECIPE_NO_UPDATE_REASON = "Version 2.0 is unstable"
-
-You can check the detail information at:
-
-http://recipes.yoctoproject.org/
-
-"""
-
     upgradable_count = 0
     no_upgradable_count = 0
     maintainers = Maintainer.objects.all().order_by("name")
 
-    table_headers = ['Package', 'Version', 'Upstream version',
+    RecipeUpgradeLine = namedtuple('RecipeUpgradeLine', ['pn', 'pv', 'pv_upstream', 'maintainer', 'noupgradereason'])
-                'Maintainer', 'NoUpgradeReason']
+
-    table = []
+    recipelines = []
     for m in maintainers:
         for recipe in recipes.keys():
             recipe_maintainer = recipes[recipe]['maintainer']
@@ -69,50 +56,66 @@ http://recipes.yoctoproject.org/
                 name_max_len = 20
                 reason_max_len = 30
 
-                pn = recipe.pn
+                recipelines.append(RecipeUpgradeLine(recipe.pn, recipe.pv, recipe_upstream.version, m.name, recipe_upstream.no_update_reason))
-                if len(pn) > pn_max_len:
-                    pn = pn[0:pn_max_len - 3] + "..."
-
-                pv = recipe.pv
-                if len(pv) > pv_max_len:
-                    pv = pv[0:pv_max_len - 3] + "..."
-
-                pv_up = recipe_upstream.version
-                if len(pv_up) > pv_max_len:
-                    pv_up = pv_up[0:pv_max_len - 3] + "..."
-
-                name = m.name
-                if len(name) > name_max_len:
-                    name = name[0:name_max_len - 3] + "..."
-
-                reason = recipe_upstream.no_update_reason
-                if len(reason) > reason_max_len:
-                    reason = reason[0:reason_max_len - 3] + "..."
-
-                table.append([pn, pv, pv_up, name, reason])
 
                 upgradable_count = upgradable_count + 1
                 if recipe_upstream.no_update_reason:
                     no_upgradable_count = no_upgradable_count + 1
 
-    body = tabulate(table, table_headers, tablefmt="simple")
 
-    footer = """
+    commits = []
-\nUpgradable count: %d\nUpgradable total count: %d\n
+    fetchdir = settings.LAYER_FETCH_DIR
-The based commit is:
+    for item in maintplan.maintenanceplanlayerbranch_set.all():
+        layerbranch = item.layerbranch
+        layer = layerbranch.layer
+        urldir = layer.get_fetch_dir()
+        repodir = os.path.join(fetchdir, urldir)
+        # FIXME this assumes the correct branch is checked out
+        topcommitdesc = utils.runcmd("git log -1 --oneline", repodir).strip()
+        commits.append('%s: %s' % (layerbranch.layer.name, topcommitdesc))
 
-%s
+    # Render the subject as a template (to allow a bit of flexibility)
-Any problem, please contact Anibal Limon <anibal.limon@intel.com> 
+    subject = options.subject or maintplan.email_subject
-""" % ((upgradable_count - no_upgradable_count), upgradable_count, 
+    subject_template = Template(subject)
-        utils.runcmd("git log -1", repodir))
+    current_site = Site.objects.get_current()
+    d = Context({
+        'maintplan': maintplan,
+        'site': current_site,
+    })
+    subject_content = subject_template.render(d)
 
-    #
+    from_email = options._from or maintplan.email_from
-    subject = options.subject
+    if options.to:
-    from_email = options._from
+        to_email_list = options.to.split(';')
-    to_email_list = options.to.split(';')
+    elif maintplan.email_to:
-    text_content = header + body + footer
+        to_email_list = maintplan.email_to.split(';')
+    else:
+        to_email_list = []
 
-    msg = EmailMessage(subject, text_content, from_email, to_email_list)
+    if not subject:
+        logger.error('No subject specified in maintenance plan %s and none specified on command line' % maintplan.name)
+        sys.exit(1)
+    if not from_email:
+        logger.error('No sender email address specified in maintenance plan %s and none specified on command line' % maintplan.name)
+        sys.exit(1)
+    if not to_email_list:
+        logger.error('No recipient email address specified in maintenance plan %s and none specified on command line' % maintplan.name)
+        sys.exit(1)
+
+    plaintext = get_template('rrs/report_email.txt')
+    site_url = 'http://' + current_site.domain + reverse('rrs_frontpage')
+    d = Context({
+        'maintplan': maintplan,
+        'site': current_site,
+        'site_url': site_url,
+        'upgradable_count': (upgradable_count - no_upgradable_count),
+        'total_upgradable_count': upgradable_count,
+        'commits': commits,
+        'recipelines': recipelines,
+    })
+    text_content = plaintext.render(d)
+
+    msg = EmailMessage(subject_content, text_content, from_email, to_email_list)
     msg.send()
 
 def main():
@@ -120,14 +123,12 @@ def main():
         usage = """
     %prog [options]""")
 
-    parser.add_option("-l", "--layername",
-            action="store", dest="layername", default=settings.CORE_LAYER_NAME)
     parser.add_option("-s", "--subject",
-            action="store", dest="subject", default=settings.RRS_EMAIL_SUBJECT)
+            action="store", dest="subject", help='Override email subject')
     parser.add_option("-f", "--from",
-            action="store", dest="_from", default=settings.RRS_EMAIL_FROM)
+            action="store", dest="_from", help='Override sender address')
     parser.add_option("-t", "--to",
-            action="store", dest="to", default=settings.RRS_EMAIL_TO)
+            action="store", dest="to", help='Override recipient address')
     parser.add_option("-d", "--debug",
             help = "Enable debug output",
             action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
@@ -137,36 +138,40 @@ def main():
 
     options, args = parser.parse_args(sys.argv)
 
-    recipes = {}
-    layer = LayerItem.objects.filter(name = options.layername)[0]
-
     # get recipes for send email
-    layerbranch = layer.get_layerbranch(LAYERBRANCH_NAME)
+    maintplans = MaintenancePlan.objects.filter(email_enabled=True)
-    recipe_upstream_history = RecipeUpstreamHistory.get_last()
+    if not maintplans.exists():
-    if recipe_upstream_history is None:
+        logger.error('No maintenance plans with email enabled were found')
-        logger.warn('I don\'t have Upstream information yet, run update.py script')
         sys.exit(1)
 
-    recipe_maintainer_history = RecipeMaintainerHistory.get_last()
+    for maintplan in maintplans:
-    if recipe_maintainer_history is None:
+        recipes = {}
-        logger.warn('I don\'t have Maintainership information yet,' +
+        for item in maintplan.maintenanceplanlayerbranch_set.all():
-                ' run rrs_maintainer_history.py script')
+            layerbranch = item.layerbranch
-        sys.exit(1)
 
-    for recipe in Recipe.objects.filter(layerbranch = layerbranch):
+            recipe_upstream_history = RecipeUpstreamHistory.get_last()
-        recipe_upstream_query = RecipeUpstream.objects.filter(recipe =
+            if recipe_upstream_history is None:
-                recipe, history = recipe_upstream_history)
+                logger.warn('I don\'t have Upstream information yet, run update.py script')
-        if recipe_upstream_query and recipe_upstream_query[0].status == 'N':
+                sys.exit(1)
-            recipes[recipe] = {}
 
-            recipe_maintainer = RecipeMaintainer.objects.filter(recipe =
+            recipe_maintainer_history = RecipeMaintainerHistory.get_last()
-                    recipe, history = recipe_maintainer_history)[0]
+            if recipe_maintainer_history is None:
-            recipes[recipe]['maintainer'] = recipe_maintainer
+                logger.warn('I don\'t have Maintainership information yet,' +
-            recipes[recipe]['upstream'] = recipe_upstream_query[0]
+                        ' run rrs_maintainer_history.py script')
+                sys.exit(1)
 
-    repodir = os.path.join(settings.LAYER_FETCH_DIR, layer.get_fetch_dir())
+            for recipe in layerbranch.recipe_set.all():
+                recipe_upstream_query = RecipeUpstream.objects.filter(recipe =
+                        recipe, history = recipe_upstream_history)
+                if recipe_upstream_query and recipe_upstream_query[0].status == 'N':
+                    recipes[recipe] = {}
 
-    send_email(recipes, repodir, options)
+                    recipe_maintainer = RecipeMaintainer.objects.filter(recipe =
+                            recipe, history = recipe_maintainer_history)[0]
+                    recipes[recipe]['maintainer'] = recipe_maintainer
+                    recipes[recipe]['upstream'] = recipe_upstream_query[0]
+
+        send_email(maintplan, recipes, options)
 
 if __name__ == "__main__":
     main()

settings.py

@@ -234,10 +234,5 @@ RABBIT_BACKEND = 'rpc://'
 # Used for fetching repo
 PARALLEL_JOBS = "4"
 
-# Settings for Recipe reporting system
-RRS_EMAIL_SUBJECT = '[Recipe reporting system] Upgradable recipe name list'
-RRS_EMAIL_FROM = 'recipe-report@yoctoproject.org'
-RRS_EMAIL_TO = 'list@example.com'
-
 # Full path to directory where rrs tools stores logs
 TOOLS_LOG_DIR = ""

templates/rrs/report_email.txt

@@ -0,0 +1,26 @@
+This is an automated message from the Recipe Reporting System.
+
+Listed below are recipes which need to be upgraded based on the versions
+available upstream. If a maintainer believes some of them cannot be upgraded
+at this time, they can set RECIPE_NO_UPDATE_REASON in the recipe file(s) as
+a reminder of why not.
+
+Example:
+RECIPE_NO_UPDATE_REASON = "Version 2.0 is unstable"
+
+For further details please visit:
+
+  {{ site_url }}
+
+Package               Version          Upstream version      Maintainer            No upgrade reason
+--------------------  ---------------  --------------------  --------------------  ------------------------------
+{% for line in recipelines %}{{ line.pn|truncatechars:20|ljust:20 }}  {{ line.pv|truncatechars:15|ljust:15 }}  {{ line.pv_upstream|truncatechars:20|ljust:20 }}  {{ line.maintainer|truncatechars:20|ljust:20 }}  {{ line.noupgradereason|truncatechars:30 }}
+{% endfor %}
+Upgradable count: {{ upgradable_count }}
+Upgradable total count: {{ total_upgradable_count }}
+
+Based upon current {{ commits|length|pluralize:"commit,commits" }}:
+
+{% for commit in commits %}  {{ commit }}
+{% endfor %}
+{% if maintplan.admin %}If there are any problems with the above, please contact: {{ maintplan.admin.first_name }} {{ maintplan.admin.last_name }} <{{ maintplan.admin.email }}>{% endif %}