diff --git a/layerindex/tools/import_classic_wiki.py b/layerindex/tools/import_classic_wiki.py deleted file mode 100755 index bfebcd9..0000000 --- a/layerindex/tools/import_classic_wiki.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python - -# Import migration info from OE-Classic recipes wiki page into OE -# layer index database -# -# Copyright (C) 2013 Intel Corporation -# Author: Paul Eggleton -# -# Licensed under the MIT license, see COPYING.MIT for details - -import sys -import os.path - -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) - -import optparse -import re -import utils -import logging - -logger = utils.logger_create('LayerIndexImport') - - -class DryRunRollbackException(Exception): - pass - - -def read_page(site, path): - ret = {} - import httplib - conn = httplib.HTTPConnection(site) - conn.request("GET", path) - resp = conn.getresponse() - if resp.status in [200, 302]: - data = resp.read() - in_table = False - for line in data.splitlines(): - if line.startswith('{|'): - in_table = True - continue - if in_table: - if line.startswith('|}'): - # We're done - in_table = False - continue - elif line.startswith('!'): - pass - elif not line.startswith('|-'): - if line.startswith("|| ''"): - continue - fields = line.split('||') - pn = fields[0].strip('|[]').split()[1] - comment = fields[1] - if comment: - ret[pn] = comment - else: - logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) - return ret - -def main(): - - parser = optparse.OptionParser( - usage = """ - %prog [options]""") - - parser.add_option("-b", "--branch", - help = "Specify branch to import into", - action="store", dest="branch", default='oe-classic') - parser.add_option("-l", "--layer", - help = "Specify layer to import into", - action="store", dest="layer", default='oe-classic') - parser.add_option("-n", "--dry-run", - help = "Don't write any data back to the database", - action="store_true", dest="dryrun") - parser.add_option("-d", "--debug", - help = "Enable debug output", - action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) - parser.add_option("-q", "--quiet", - help = "Hide all output except error messages", - action="store_const", const=logging.ERROR, dest="loglevel") - - options, args = parser.parse_args(sys.argv) - - utils.setup_django() - from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe - from django.db import transaction - - logger.setLevel(options.loglevel) - - res = list(LayerItem.objects.filter(name=options.layer)[:1]) - if res: - layer = res[0] - else: - logger.error('Specified layer %s does not exist in database' % options.layer) - sys.exit(1) - - layerbranch = layer.get_layerbranch(options.branch) - if not layerbranch: - logger.error("Specified branch %s does not exist in database" % options.branch) - sys.exit(1) - - recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw") - recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw") - - try: - with transaction.atomic(): - recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) - for pn, comment in recipes.items(): - newpn = '' - newlayer = '' - status = 'U' - comment = comment.strip(' -') - if 'provided by' in comment: - res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - if pn.endswith('-native') or pn.endswith('-cross'): - status = 'P' - else: - status = 'R' - elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: - res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'R' - elif 'obsolete' in comment or 'superseded' in comment: - res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - elif comment.startswith('superseded by'): - comment = comment[14:] - elif comment.startswith('obsolete'): - comment = comment[9:] - status = 'O' - elif 'PACKAGECONFIG' in comment: - res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'C' - - if newlayer: - if newlayer.lower() == 'oe-core': - newlayer = 'openembedded-core' - - # Remove all links from comments because they'll be picked up as categories - comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) - # Split out categories - categories = re.findall(r'\[([^]]*)\]', comment) - for cat in categories: - comment = comment.replace('[%s]' % cat, '') - if '(GPE)' in comment or pn.startswith('gpe'): - categories.append('GPE') - comment = comment.replace('(GPE)', '') - - comment = comment.strip('- ') - - logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) - - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn).filter(deleted=False) - if recipequery: - for recipe in recipequery: - recipe.cover_layerbranch = None - if newlayer: - res = list(LayerItem.objects.filter(name=newlayer)[:1]) - if res: - newlayeritem = res[0] - recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') - else: - logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) - recipe.cover_pn = newpn - recipe.cover_status = status - recipe.cover_verified = True - recipe.cover_comment = comment - recipe.classic_category = " ".join(categories) - recipe.save() - else: - logger.info('No OE-Classic recipe with name "%s" count be found' % pn) - sys.exit(1) - - if options.dryrun: - raise DryRunRollbackException() - except DryRunRollbackException: - pass - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/layerindex/tools/import_wiki_layers.py b/layerindex/tools/import_wiki_layers.py deleted file mode 100755 index 71f26ea..0000000 --- a/layerindex/tools/import_wiki_layers.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -# Import layer index wiki page into database -# -# Copyright (C) 2013 Intel Corporation -# Author: Paul Eggleton -# -# Licensed under the MIT license, see COPYING.MIT for details - - -import sys -import os.path - -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) - -import optparse -import re -import utils - -logger = utils.logger_create('LayerIndexImport') - - -class DryRunRollbackException(Exception): - pass - - -def main(): - - parser = optparse.OptionParser( - usage = """ - %prog [options]""") - - options, args = parser.parse_args(sys.argv) - - utils.setup_django() - from layerindex.models import LayerItem, LayerBranch, LayerDependency - from django.db import transaction - - import httplib - conn = httplib.HTTPConnection("www.openembedded.org") - conn.request("GET", "/wiki/LayerIndex?action=raw") - resp = conn.getresponse() - if resp.status in [200, 302]: - data = resp.read() - in_table = False - layer_type = 'M' - nowiki_re = re.compile(r'') - link_re = re.compile(r'\[(http.*) +link\]') - readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') - master_branch = utils.get_branch('master') - core_layer = None - with transaction.atomic(): - for line in data.splitlines(): - if line.startswith('{|'): - in_table = True - continue - if in_table: - if line.startswith('|}'): - # We're done - break - elif line.startswith('!'): - section = line.split('|', 1)[1].strip("'") - if section.startswith('Base'): - layer_type = 'A' - elif section.startswith('Board'): - layer_type = 'B' - elif section.startswith('Software'): - layer_type = 'S' - elif section.startswith('Distribution'): - layer_type = 'D' - else: - layer_type = 'M' - elif not line.startswith('|-'): - if line.startswith("|| ''"): - continue - fields = line.split('||') - layer = LayerItem() - layer.name = fields[1].strip() - if ' ' in layer.name: - logger.warn('Skipping layer %s - name invalid' % layer.name) - continue - logger.info('Adding layer %s' % layer.name) - layer.status = 'P' - layer.layer_type = layer_type - layer.summary = fields[2].strip() - layer.description = layer.summary - if len(fields) > 6: - res = link_re.match(fields[6].strip()) - if res: - link = res.groups(1)[0].strip() - if link.endswith('/README') or readme_re.search(link): - link = 'README' - layer.usage_url = link - - repoval = nowiki_re.sub('', fields[4]).strip() - layer.vcs_url = repoval - if repoval.startswith('git://git.openembedded.org/'): - reponame = re.sub('^.*/', '', repoval) - layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame - layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%' - elif repoval.startswith('git://git.yoctoproject.org/'): - reponame = re.sub('^.*/', '', repoval) - layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame - layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%' - elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'): - reponame = re.sub('^.*github.com/', '', repoval) - reponame = re.sub('.git$', '', reponame) - layer.vcs_web_url = 'http://github.com/' + reponame - layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/' - layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/' - layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%' - elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'): - reponame = re.sub('^.*gitlab.com/', '', repoval) - reponame = re.sub('.git$', '', reponame) - layer.vcs_web_url = 'http://gitlab.com/' + reponame - layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/' - layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/' - layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%' - elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'): - reponame = re.sub('^.*bitbucket.org/', '', repoval) - reponame = re.sub('.git$', '', reponame) - layer.vcs_web_url = 'http://bitbucket.org/' + reponame - layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' - layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' - layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%' - elif '.git' in repoval: - res = link_re.match(fields[5].strip()) - layer.vcs_web_url = res.groups(1)[0] - layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url) - layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url) - layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url) - - layer.save() - layerbranch = LayerBranch() - layerbranch.layer = layer - layerbranch.branch = master_branch - layerbranch.vcs_subdir = fields[3].strip() - layerbranch.save() - if layer.name != 'openembedded-core': - if not core_layer: - core_layer = utils.get_layer('openembedded-core') - if core_layer: - layerdep = LayerDependency() - layerdep.layerbranch = layerbranch - layerdep.dependency = core_layer - layerdep.save() - else: - logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) - - sys.exit(0) - - -if __name__ == "__main__": - main()