mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-05 05:04:46 +02:00
c3a8eb4d82
Add a script for doing a one-time import of OE-Classic recipe information, so comparisons against OE-Core can be performed; this is stored using a new ClassicRecipe model supporting additional fields for tracking migration status. The migration status fields can be updated as well as viewed and summarised in graph format. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
204 lines
7.5 KiB
Python
204 lines
7.5 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Import migration info from OE-Classic recipes wiki page into OE
|
|
# layer index database
|
|
#
|
|
# Copyright (C) 2013 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
import sys
|
|
import os.path
|
|
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
|
|
|
|
import optparse
|
|
import re
|
|
import utils
|
|
import logging
|
|
|
|
logger = utils.logger_create('LayerIndexImport')
|
|
|
|
|
|
def read_page(site, path):
|
|
ret = {}
|
|
import httplib
|
|
conn = httplib.HTTPConnection(site)
|
|
conn.request("GET", path)
|
|
resp = conn.getresponse()
|
|
if resp.status in [200, 302]:
|
|
data = resp.read()
|
|
in_table = False
|
|
for line in data.splitlines():
|
|
if line.startswith('{|'):
|
|
in_table = True
|
|
continue
|
|
if in_table:
|
|
if line.startswith('|}'):
|
|
# We're done
|
|
in_table = False
|
|
continue
|
|
elif line.startswith('!'):
|
|
pass
|
|
elif not line.startswith('|-'):
|
|
if line.startswith("|| ''"):
|
|
continue
|
|
fields = line.split('||')
|
|
pn = fields[0].strip('|[]').split()[1]
|
|
comment = fields[1]
|
|
if comment:
|
|
ret[pn] = comment
|
|
else:
|
|
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
|
|
return ret
|
|
|
|
def main():
|
|
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options]""")
|
|
|
|
parser.add_option("-b", "--branch",
|
|
help = "Specify branch to import into",
|
|
action="store", dest="branch", default='oe-classic')
|
|
parser.add_option("-l", "--layer",
|
|
help = "Specify layer to import into",
|
|
action="store", dest="layer", default='oe-classic')
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
utils.setup_django()
|
|
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe
|
|
from django.db import transaction
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
res = list(LayerItem.objects.filter(name=options.layer)[:1])
|
|
if res:
|
|
layer = res[0]
|
|
else:
|
|
logger.error('Specified layer %s does not exist in database' % options.layer)
|
|
sys.exit(1)
|
|
|
|
layerbranch = layer.get_layerbranch(options.branch)
|
|
if not layerbranch:
|
|
logger.error("Specified branch %s does not exist in database" % options.branch)
|
|
sys.exit(1)
|
|
|
|
recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw")
|
|
recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw")
|
|
|
|
transaction.enter_transaction_management()
|
|
transaction.managed(True)
|
|
try:
|
|
recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
|
|
for pn, comment in recipes.items():
|
|
newpn = ''
|
|
newlayer = ''
|
|
status = 'U'
|
|
comment = comment.strip(' -')
|
|
if 'provided by' in comment:
|
|
res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
if pn.endswith('-native') or pn.endswith('-cross'):
|
|
status = 'P'
|
|
else:
|
|
status = 'R'
|
|
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
|
|
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if not res:
|
|
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if not res:
|
|
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
status = 'R'
|
|
elif 'obsolete' in comment or 'superseded' in comment:
|
|
res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
elif comment.startswith('superseded by'):
|
|
comment = comment[14:]
|
|
elif comment.startswith('obsolete'):
|
|
comment = comment[9:]
|
|
status = 'O'
|
|
elif 'PACKAGECONFIG' in comment:
|
|
res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
status = 'C'
|
|
|
|
if newlayer:
|
|
if newlayer.lower() == 'oe-core':
|
|
newlayer = 'openembedded-core'
|
|
|
|
# Remove all links from comments because they'll be picked up as categories
|
|
comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
|
|
# Split out categories
|
|
categories = re.findall(r'\[([^]]*)\]', comment)
|
|
for cat in categories:
|
|
comment = comment.replace('[%s]' % cat, '')
|
|
if '(GPE)' in comment or pn.startswith('gpe'):
|
|
categories.append('GPE')
|
|
comment = comment.replace('(GPE)', '')
|
|
|
|
comment = comment.strip('- ')
|
|
|
|
logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))
|
|
|
|
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn)
|
|
if recipequery:
|
|
for recipe in recipequery:
|
|
recipe.cover_layerbranch = None
|
|
if newlayer:
|
|
res = list(LayerItem.objects.filter(name=newlayer)[:1])
|
|
if res:
|
|
newlayeritem = res[0]
|
|
recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
|
|
else:
|
|
logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
|
|
recipe.cover_pn = newpn
|
|
recipe.cover_status = status
|
|
recipe.cover_verified = True
|
|
recipe.cover_comment = comment
|
|
recipe.classic_category = " ".join(categories)
|
|
recipe.save()
|
|
else:
|
|
logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
|
|
sys.exit(1)
|
|
|
|
if options.dryrun:
|
|
transaction.rollback()
|
|
else:
|
|
transaction.commit()
|
|
except:
|
|
transaction.rollback()
|
|
raise
|
|
finally:
|
|
transaction.leave_transaction_management()
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|