mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 12:49:01 +02:00
87e27a26d5
Add a script to import "recipe" information from other distro metadata, based on the import_classic.py script. At the moment, this assumes a directory where each subdirectory is a package directory containing a spec file; this would be suitable for distributions such as Fedora assuming you have all of the package repos checked out locally. Since you can add additional information to these records (the cover fields pointing to matching recipes), existing records are updated rather than deleting everything and re-importing, and we only mark records as deleted rather than actually deleting them (in case you accidentally point the script at an empty directory or similar). Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
202 lines
7.8 KiB
Python
Executable File
202 lines
7.8 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Import migration info from OE-Classic recipes wiki page into OE
|
|
# layer index database
|
|
#
|
|
# Copyright (C) 2013 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
import sys
|
|
import os.path
|
|
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
|
|
|
|
import optparse
|
|
import re
|
|
import utils
|
|
import logging
|
|
|
|
logger = utils.logger_create('LayerIndexImport')
|
|
|
|
|
|
class DryRunRollbackException(Exception):
|
|
pass
|
|
|
|
|
|
def read_page(site, path):
|
|
ret = {}
|
|
import httplib
|
|
conn = httplib.HTTPConnection(site)
|
|
conn.request("GET", path)
|
|
resp = conn.getresponse()
|
|
if resp.status in [200, 302]:
|
|
data = resp.read()
|
|
in_table = False
|
|
for line in data.splitlines():
|
|
if line.startswith('{|'):
|
|
in_table = True
|
|
continue
|
|
if in_table:
|
|
if line.startswith('|}'):
|
|
# We're done
|
|
in_table = False
|
|
continue
|
|
elif line.startswith('!'):
|
|
pass
|
|
elif not line.startswith('|-'):
|
|
if line.startswith("|| ''"):
|
|
continue
|
|
fields = line.split('||')
|
|
pn = fields[0].strip('|[]').split()[1]
|
|
comment = fields[1]
|
|
if comment:
|
|
ret[pn] = comment
|
|
else:
|
|
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
|
|
return ret
|
|
|
|
def main():
|
|
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options]""")
|
|
|
|
parser.add_option("-b", "--branch",
|
|
help = "Specify branch to import into",
|
|
action="store", dest="branch", default='oe-classic')
|
|
parser.add_option("-l", "--layer",
|
|
help = "Specify layer to import into",
|
|
action="store", dest="layer", default='oe-classic')
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
utils.setup_django()
|
|
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe
|
|
from django.db import transaction
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
res = list(LayerItem.objects.filter(name=options.layer)[:1])
|
|
if res:
|
|
layer = res[0]
|
|
else:
|
|
logger.error('Specified layer %s does not exist in database' % options.layer)
|
|
sys.exit(1)
|
|
|
|
layerbranch = layer.get_layerbranch(options.branch)
|
|
if not layerbranch:
|
|
logger.error("Specified branch %s does not exist in database" % options.branch)
|
|
sys.exit(1)
|
|
|
|
recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw")
|
|
recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw")
|
|
|
|
try:
|
|
with transaction.atomic():
|
|
recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
|
|
for pn, comment in recipes.items():
|
|
newpn = ''
|
|
newlayer = ''
|
|
status = 'U'
|
|
comment = comment.strip(' -')
|
|
if 'provided by' in comment:
|
|
res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
if pn.endswith('-native') or pn.endswith('-cross'):
|
|
status = 'P'
|
|
else:
|
|
status = 'R'
|
|
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
|
|
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if not res:
|
|
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if not res:
|
|
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
status = 'R'
|
|
elif 'obsolete' in comment or 'superseded' in comment:
|
|
res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
elif comment.startswith('superseded by'):
|
|
comment = comment[14:]
|
|
elif comment.startswith('obsolete'):
|
|
comment = comment[9:]
|
|
status = 'O'
|
|
elif 'PACKAGECONFIG' in comment:
|
|
res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
if res:
|
|
newpn = res.group(1)
|
|
newlayer = res.group(2)
|
|
comment = res.group(3)
|
|
status = 'C'
|
|
|
|
if newlayer:
|
|
if newlayer.lower() == 'oe-core':
|
|
newlayer = 'openembedded-core'
|
|
|
|
# Remove all links from comments because they'll be picked up as categories
|
|
comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
|
|
# Split out categories
|
|
categories = re.findall(r'\[([^]]*)\]', comment)
|
|
for cat in categories:
|
|
comment = comment.replace('[%s]' % cat, '')
|
|
if '(GPE)' in comment or pn.startswith('gpe'):
|
|
categories.append('GPE')
|
|
comment = comment.replace('(GPE)', '')
|
|
|
|
comment = comment.strip('- ')
|
|
|
|
logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))
|
|
|
|
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn).filter(deleted=False)
|
|
if recipequery:
|
|
for recipe in recipequery:
|
|
recipe.cover_layerbranch = None
|
|
if newlayer:
|
|
res = list(LayerItem.objects.filter(name=newlayer)[:1])
|
|
if res:
|
|
newlayeritem = res[0]
|
|
recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
|
|
else:
|
|
logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
|
|
recipe.cover_pn = newpn
|
|
recipe.cover_status = status
|
|
recipe.cover_verified = True
|
|
recipe.cover_comment = comment
|
|
recipe.classic_category = " ".join(categories)
|
|
recipe.save()
|
|
else:
|
|
logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
|
|
sys.exit(1)
|
|
|
|
if options.dryrun:
|
|
raise DryRunRollbackException()
|
|
except DryRunRollbackException:
|
|
pass
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|