layerindex-web/rrs/tools/rrs_upstream_history.py

#!/usr/bin/env python3

# Get upstream information for recipes.
#
# Copyright (C) 2015 Intel Corporation
# Author: Anibal Limon <anibal.limon@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details

import sys
import os.path
import optparse
import logging
from datetime import datetime
import shutil

sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__))))
from common import common_setup, load_recipes, \
        get_pv_type, get_logger, DryRunRollbackException
common_setup()
from layerindex import utils

utils.setup_django()
from django.db import transaction
from django.core.exceptions import ObjectDoesNotExist
import settings

logger = get_logger("UpstreamHistory", settings)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
    logger.error("Please set LAYER_FETCH_DIR in settings.py")
    sys.exit(1)

# setup bitbake path
bitbakepath = os.path.join(fetchdir, 'bitbake')
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))

from layerindex.models import Recipe, LayerBranch
from rrs.models import RecipeUpstream, RecipeUpstreamHistory, MaintenancePlan

def set_regexes(d):
    """
        Utility function to set regexes to SPECIAL_PKGSUFFIX packages
        that don't have set it.

        For example: python-native use regex from python if don't have
        one set it.
    """

    variables = ('REGEX', 'REGEX_URI', 'GITTAGREGEX')

    if any(d.getVar(var, True) for var in variables):
        return

    suffixes = d.getVar('SPECIAL_PKGSUFFIX', True).split()
    prefixes = ['nativesdk-']

    special = list(suffixes)
    special.extend(prefixes)

    localdata = bb.data.createCopy(d)
    pn = localdata.getVar('PN', True)
    for s in special:
        if pn.find(s) != -1:
            if s in suffixes:
                pnstripped = pn.split(s)[0]
            else:
                pnstripped = pn.replace(s, '')

            localdata.setVar('OVERRIDES', "pn-" + pnstripped + ":" +
                    d.getVar('OVERRIDES', True))
            bb.data.update_data(localdata)

            for var in variables:
                new_value = localdata.getVar(var, True)
                if new_value is None:
                    continue

                d.setVar(var, new_value)
                logger.debug("%s: %s new value %s" % (pn, var,
                    d.getVar(var, True)))
            break

def get_upstream_info(layerbranch, recipe_data, result):
    from bb.utils import vercmp_string
    from oe.recipeutils import get_recipe_upstream_version, \
            get_recipe_pv_without_srcpv

    pn = recipe_data.getVar('PN', True)
    recipes = Recipe.objects.filter(layerbranch=layerbranch, pn=pn)
    if not recipes:
        logger.warning("%s: in layer branch %s not found." % \
                (pn, str(layerbranch)))
        return
    recipe = recipes[0]

    ru = RecipeUpstream()
    ru.recipe = recipe

    ru_info = None
    try:
        ru_info = get_recipe_upstream_version(recipe_data)
    except Exception as e:
        logger.exception("%s: in layer branch %s, %s" % (pn,
            str(layerbranch), str(e)))

    if ru_info is not None and ru_info['version']:
        ru.version = ru_info['version']
        ru.type = ru_info['type']
        ru.date = ru_info['datetime']

        pv, _, _ = get_recipe_pv_without_srcpv(recipe.pv, 
                get_pv_type(recipe.pv)) 
        upv, _, _ = get_recipe_pv_without_srcpv(ru_info['version'],
                get_pv_type(ru_info['version']))

        cmp_ver = vercmp_string(pv, upv)
        if cmp_ver == -1:
            ru.status = 'N' # Not update
        elif cmp_ver == 0:
            ru.status = 'Y' # Up-to-date
        elif cmp_ver == 1:
            ru.status = 'D' # Downgrade, need to review why
    else:
        ru.version = ''
        ru.type = 'M'
        ru.date = datetime.now()
        ru.status = 'U' # Unknown

    ru.no_update_reason = recipe_data.getVar('RECIPE_NO_UPDATE_REASON',
            True) or ''

    result.append((recipe, ru))

if __name__=="__main__":
    parser = optparse.OptionParser(usage = """%prog [options]""")

    parser.add_option("-p", "--plan",
            help="Specify maintenance plan to operate on (default is all plans that have updates enabled)",
            action="store", dest="plan", default=None)

    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    
    parser.add_option("--dry-run",
            help = "Do not write any data back to the database",
            action="store_true", dest="dry_run", default=False)

    options, args = parser.parse_args(sys.argv)
    logger.setLevel(options.loglevel)

    if options.plan:
        maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
        if not maintplans.exists():
            logger.error('No maintenance plan with ID %s found' % options.plan)
            sys.exit(1)
    else:
        maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
        if not maintplans.exists():
            logger.error('No enabled maintenance plans found')
            sys.exit(1)

    logger.debug("Starting upstream history...")

    lockfn = os.path.join(fetchdir, "layerindex.lock")
    lockfile = utils.lock_file(lockfn)
    if not lockfile:
        logger.error("Layer index lock timeout expired")
        sys.exit(1)
    try:
        origsyspath = sys.path

        for maintplan in maintplans:
            for item in maintplan.maintenanceplanlayerbranch_set.all():
                layerbranch = item.layerbranch
                with transaction.atomic():
                    sys.path = origsyspath

                    layer = layerbranch.layer
                    urldir = layer.get_fetch_dir()
                    repodir = os.path.join(fetchdir, urldir)
                    layerdir = os.path.join(repodir, layerbranch.vcs_subdir)

                    recipe_files = []
                    for recipe in layerbranch.recipe_set.all():
                        file = str(os.path.join(layerdir, recipe.full_path()))
                        recipe_files.append(file)

                    (tinfoil, d, recipes, tempdir) = load_recipes(layerbranch, bitbakepath,
                            fetchdir, settings, logger,  recipe_files=recipe_files)
                    try:

                        if not recipes:
                            continue

                        utils.setup_core_layer_sys_path(settings, layerbranch.branch.name)

                        for recipe_data in recipes:
                            set_regexes(recipe_data)

                        history = RecipeUpstreamHistory(start_date = datetime.now())

                        result = []
                        for recipe_data in recipes:
                            get_upstream_info(layerbranch, recipe_data, result)

                        history.end_date = datetime.now()
                        history.save()

                        for res in result:
                            (recipe, ru) = res

                            ru.history = history
                            ru.save()

                            logger.debug('%s: layer branch %s, pv %s, upstream (%s)' % (recipe.pn,
                                str(layerbranch), recipe.pv, str(ru)))

                    finally:
                        tinfoil.shutdown()
                        shutil.rmtree(tempdir)
                    if options.dry_run:
                        raise DryRunRollbackException
    except DryRunRollbackException:
        pass
    finally:
        utils.unlock_file(lockfile)