layerindex-web/scripts/update.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Fetch layer repositories and update layer index database
#
# Copyright (C) 2013 - 2015 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
# Contributor: Aníbal Limón <anibal.limon@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details

import sys
import os.path
import shutil

sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../lib')))

import optparse
import logging
import utils

utils.setup_django()
import settings

import warnings
warnings.filterwarnings("ignore", category = DeprecationWarning)

logger = utils.logger_create('LayerindexUpdate')

from layerindex_update import LayerindexUpdater

def main():
    parser = optparse.OptionParser(
        usage = """
    %prog [options]""")

    parser.add_option("-b", "--branch",
            help = "Specify branch to update",
            action="store", dest="branch", default='master')
    parser.add_option("-l", "--layer",
            help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
            action="store", dest="layers")
    parser.add_option("-r", "--reload",
            help = "Reload recipe data instead of updating since last update",
            action="store_true", dest="reload")
    parser.add_option("", "--fullreload",
            help = "Discard existing recipe data and fetch it from scratch",
            action="store_true", dest="fullreload")
    parser.add_option("-n", "--dry-run",
            help = "Don't write any data back to the database",
            action="store_true", dest="dryrun")
    parser.add_option("-x", "--nofetch",
            help = "Don't fetch repositories",
            action="store_true", dest="nofetch")
    parser.add_option("", "--nocheckout",
            help = "Don't check out branches",
            action="store_true", dest="nocheckout")
    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    parser.add_option("-q", "--quiet",
            help = "Hide all output except error messages",
            action="store_const", const=logging.ERROR, dest="loglevel")

    options, args = parser.parse_args(sys.argv)
    if len(args) > 1:
        logger.error('unexpected argument "%s"' % args[1])
        parser.print_help()
        sys.exit(1)

    if options.fullreload:
        options.reload = True

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)
    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)

    lockfn = os.path.join(fetchdir, "layerindex.lock")
    lockfile = utils.lock_file(lockfn)
    if not lockfile:
        logger.error("Layer index lock timeout expired")
        sys.exit(1)

    bitbakepath = update_repo(fetchdir, 'bitbake', settings.BITBAKE_REPO_URL, logger)
    (layerquery, fetchedrepos, failedrepos) = update_layers(options, fetchdir, logger)
    (tinfoil, tempdir) = get_tinfoil(branch, bitbakepath, options, logger)

    layerindex_updater = LayerindexUpdater(options, fetchdir, layerquery, fetchedrepos,
            failedrepos, logger)
    layerindex_updater.run(tinfoil)

    shutil.rmtree(tempdir)
    utils.unlock_file(lockfile)

def update_repo(fetchdir, repo_name, repo_url, logger):
    path = os.path.join(fetchdir, repo_name)

    logger.info("Fetching %s from remote repository %s"
                    % (repo_name, repo_url))
    if not os.path.exists(path):
        out = utils.runcmd("git clone %s %s" %
                (repo_url, repo_name), fetchdir,
                logger = logger)
    else:
        out = utils.runcmd("git fetch", path, logger = logger)

    return path

def update_layers(options, fetchdir, logger):
    from layerindex.models import LayerItem

    fetchedrepos = []
    failedrepos = []

    if options.layers:
        layerquery = LayerItem.objects.filter(classic =
                False).filter(name__in = options.layers.split(','))
    else:
        layerquery = LayerItem.objects.filter(classic =
                False).filter(status = 'P') # All published layers

    if layerquery.count() == 0:
        logger.info("No published layers to update")
        sys.exit(1)

    # Fetch latest metadata from repositories
    # Handle multiple layers in a single repo
    for layer in layerquery:
        urldir = layer.get_fetch_dir()
        repodir = os.path.join(fetchdir, urldir)

        if not (layer.vcs_url in fetchedrepos or layer.vcs_url in
                failedrepos):
            logger.info("Fetching remote repository %s" %
                    layer.vcs_url)

            out = None
            try:
                if not os.path.exists(repodir):
                    out = utils.runcmd("git clone %s %s" %
                            (layer.vcs_url, urldir), fetchdir,
                            logger = logger)
                else:
                    out = utils.runcmd("git fetch", repodir, logger =
                            logger)
            except Exception as e:
                logger.error("Fetch of layer %s failed: %s" %
                        (layer.name, str(e)))
                failedrepos.append(layer.vcs_url)
                continue

            fetchedrepos.append(layer.vcs_url)

    if not fetchedrepos:
        logger.error("No repositories could be fetched, exiting")
        sys.exit(1)

    return (layerquery, fetchedrepos, failedrepos)

def get_tinfoil(branch, bitbakepath, options, logger):
    import recipeparse
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath,
                                nocheckout = options.nocheckout, logger = logger)
    except recipeparse.RecipeParseError as e:
        logger.error(str(e))
        sys.exit(1)

    # Clear the default value of SUMMARY so that we can use DESCRIPTION instead
    # if it hasn't been set
    tinfoil.config_data.setVar('SUMMARY', '')
    # Clear the default value of DESCRIPTION so that we can see where it's not set
    tinfoil.config_data.setVar('DESCRIPTION', '')
    # Clear the default value of HOMEPAGE ('unknown')
    tinfoil.config_data.setVar('HOMEPAGE', '')
    # Set a blank value for LICENSE so that it doesn't cause the parser to die
    # (e.g. with meta-ti -, why won't they just fix that?!)
    tinfoil.config_data.setVar('LICENSE', '')

    return (tinfoil, tempdir)

if __name__ == "__main__":
    main()