layerindex-web/scripts/tools/import_classic_wiki.py

#!/usr/bin/env python

# Import migration info from OE-Classic recipes wiki page into OE
# layer index database
#
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details

import sys
import os.path

sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../../lib')))

import optparse
import re
import utils
import logging

logger = utils.logger_create('LayerIndexImport')


def read_page(site, path):
    ret = {}
    import httplib
    conn = httplib.HTTPConnection(site)
    conn.request("GET", path)
    resp = conn.getresponse()
    if resp.status in [200, 302]:
        data = resp.read()
        in_table = False
        for line in data.splitlines():
            if line.startswith('{|'):
                in_table = True
                continue
            if in_table:
                if line.startswith('|}'):
                    # We're done
                    in_table = False
                    continue
                elif line.startswith('!'):
                    pass
                elif not line.startswith('|-'):
                    if line.startswith("|| ''"):
                        continue
                    fields = line.split('||')
                    pn = fields[0].strip('|[]').split()[1]
                    comment = fields[1]
                    if comment:
                        ret[pn] = comment
    else:
        logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
    return ret

def main():

    parser = optparse.OptionParser(
        usage = """
    %prog [options]""")

    parser.add_option("-b", "--branch",
            help = "Specify branch to import into",
            action="store", dest="branch", default='oe-classic')
    parser.add_option("-l", "--layer",
            help = "Specify layer to import into",
            action="store", dest="layer", default='oe-classic')
    parser.add_option("-n", "--dry-run",
            help = "Don't write any data back to the database",
            action="store_true", dest="dryrun")
    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    parser.add_option("-q", "--quiet",
            help = "Hide all output except error messages",
            action="store_const", const=logging.ERROR, dest="loglevel")

    options, args = parser.parse_args(sys.argv)

    utils.setup_django()
    from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe
    from django.db import transaction

    logger.setLevel(options.loglevel)

    res = list(LayerItem.objects.filter(name=options.layer)[:1])
    if res:
        layer = res[0]
    else:
        logger.error('Specified layer %s does not exist in database' % options.layer)
        sys.exit(1)

    layerbranch = layer.get_layerbranch(options.branch)
    if not layerbranch:
        logger.error("Specified branch %s does not exist in database" % options.branch)
        sys.exit(1)

    recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw")
    recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw")

    transaction.enter_transaction_management()
    transaction.managed(True)
    try:
        recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
        for pn, comment in recipes.items():
            newpn = ''
            newlayer = ''
            status = 'U'
            comment = comment.strip(' -')
            if 'provided by' in comment:
                res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
                if res:
                    newpn = res.group(1)
                    newlayer = res.group(2)
                    comment = res.group(3)
                if pn.endswith('-native') or pn.endswith('-cross'):
                    status = 'P'
                else:
                    status = 'R'
            elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
                res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
                if not res:
                    res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
                if not res:
                    res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
                if res:
                    newpn = res.group(1)
                    newlayer = res.group(2)
                    comment = res.group(3)
                status = 'R'
            elif 'obsolete' in comment or 'superseded' in comment:
                res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
                if res:
                    newpn = res.group(1)
                    newlayer = res.group(2)
                    comment = res.group(3)
                elif comment.startswith('superseded by'):
                    comment = comment[14:]
                elif comment.startswith('obsolete'):
                    comment = comment[9:]
                status = 'O'
            elif 'PACKAGECONFIG' in comment:
                res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
                if res:
                    newpn = res.group(1)
                    newlayer = res.group(2)
                    comment = res.group(3)
                status = 'C'

            if newlayer:
                if newlayer.lower() == 'oe-core':
                    newlayer = 'openembedded-core'

            # Remove all links from comments because they'll be picked up as categories
            comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
            # Split out categories
            categories = re.findall(r'\[([^]]*)\]', comment)
            for cat in categories:
                comment = comment.replace('[%s]' % cat, '')
            if '(GPE)' in comment or pn.startswith('gpe'):
                categories.append('GPE')
                comment = comment.replace('(GPE)', '')

            comment = comment.strip('- ')

            logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))

            recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn)
            if recipequery:
                for recipe in recipequery:
                    recipe.cover_layerbranch = None
                    if newlayer:
                        res = list(LayerItem.objects.filter(name=newlayer)[:1])
                        if res:
                            newlayeritem = res[0]
                            recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
                        else:
                            logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
                    recipe.cover_pn = newpn
                    recipe.cover_status = status
                    recipe.cover_verified = True 
                    recipe.cover_comment = comment
                    recipe.classic_category = " ".join(categories)
                    recipe.save()
            else:
                logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
                sys.exit(1)

        if options.dryrun:
            transaction.rollback()
        else:
            transaction.commit()
    except:
        transaction.rollback()
        raise
    finally:
        transaction.leave_transaction_management()

    sys.exit(0)


if __name__ == "__main__":
    main()