From 1327fe1b425bf97a57bb9e4ccc0f95e29928c98a Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Fri, 27 Mar 2020 09:19:28 +1300 Subject: [PATCH] Drop wiki import scripts These scripts have not been used or maintained for some time; I only really kept them around in case it was useful as an example of how to poke data into the database, but we have plenty of those in other scripts. Signed-off-by: Paul Eggleton --- layerindex/tools/import_classic_wiki.py | 201 ------------------------ layerindex/tools/import_wiki_layers.py | 158 ------------------- 2 files changed, 359 deletions(-) delete mode 100755 layerindex/tools/import_classic_wiki.py delete mode 100755 layerindex/tools/import_wiki_layers.py diff --git a/layerindex/tools/import_classic_wiki.py b/layerindex/tools/import_classic_wiki.py deleted file mode 100755 index bfebcd9..0000000 --- a/layerindex/tools/import_classic_wiki.py +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env python - -# Import migration info from OE-Classic recipes wiki page into OE -# layer index database -# -# Copyright (C) 2013 Intel Corporation -# Author: Paul Eggleton -# -# Licensed under the MIT license, see COPYING.MIT for details - -import sys -import os.path - -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) - -import optparse -import re -import utils -import logging - -logger = utils.logger_create('LayerIndexImport') - - -class DryRunRollbackException(Exception): - pass - - -def read_page(site, path): - ret = {} - import httplib - conn = httplib.HTTPConnection(site) - conn.request("GET", path) - resp = conn.getresponse() - if resp.status in [200, 302]: - data = resp.read() - in_table = False - for line in data.splitlines(): - if line.startswith('{|'): - in_table = True - continue - if in_table: - if line.startswith('|}'): - # We're done - in_table = False - continue - elif line.startswith('!'): - pass - elif not line.startswith('|-'): - if line.startswith("|| ''"): - continue - fields = line.split('||') - pn = fields[0].strip('|[]').split()[1] - comment = fields[1] - if comment: - ret[pn] = comment - else: - logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) - return ret - -def main(): - - parser = optparse.OptionParser( - usage = """ - %prog [options]""") - - parser.add_option("-b", "--branch", - help = "Specify branch to import into", - action="store", dest="branch", default='oe-classic') - parser.add_option("-l", "--layer", - help = "Specify layer to import into", - action="store", dest="layer", default='oe-classic') - parser.add_option("-n", "--dry-run", - help = "Don't write any data back to the database", - action="store_true", dest="dryrun") - parser.add_option("-d", "--debug", - help = "Enable debug output", - action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) - parser.add_option("-q", "--quiet", - help = "Hide all output except error messages", - action="store_const", const=logging.ERROR, dest="loglevel") - - options, args = parser.parse_args(sys.argv) - - utils.setup_django() - from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe - from django.db import transaction - - logger.setLevel(options.loglevel) - - res = list(LayerItem.objects.filter(name=options.layer)[:1]) - if res: - layer = res[0] - else: - logger.error('Specified layer %s does not exist in database' % options.layer) - sys.exit(1) - - layerbranch = layer.get_layerbranch(options.branch) - if not layerbranch: - logger.error("Specified branch %s does not exist in database" % options.branch) - sys.exit(1) - - recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw") - recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw") - - try: - with transaction.atomic(): - recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) - for pn, comment in recipes.items(): - newpn = '' - newlayer = '' - status = 'U' - comment = comment.strip(' -') - if 'provided by' in comment: - res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - if pn.endswith('-native') or pn.endswith('-cross'): - status = 'P' - else: - status = 'R' - elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: - res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'R' - elif 'obsolete' in comment or 'superseded' in comment: - res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - elif comment.startswith('superseded by'): - comment = comment[14:] - elif comment.startswith('obsolete'): - comment = comment[9:] - status = 'O' - elif 'PACKAGECONFIG' in comment: - res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'C' - - if newlayer: - if newlayer.lower() == 'oe-core': - newlayer = 'openembedded-core' - - # Remove all links from comments because they'll be picked up as categories - comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) - # Split out categories - categories = re.findall(r'\[([^]]*)\]', comment) - for cat in categories: - comment = comment.replace('[%s]' % cat, '') - if '(GPE)' in comment or pn.startswith('gpe'): - categories.append('GPE') - comment = comment.replace('(GPE)', '') - - comment = comment.strip('- ') - - logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) - - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn).filter(deleted=False) - if recipequery: - for recipe in recipequery: - recipe.cover_layerbranch = None - if newlayer: - res = list(LayerItem.objects.filter(name=newlayer)[:1]) - if res: - newlayeritem = res[0] - recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') - else: - logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) - recipe.cover_pn = newpn - recipe.cover_status = status - recipe.cover_verified = True - recipe.cover_comment = comment - recipe.classic_category = " ".join(categories) - recipe.save() - else: - logger.info('No OE-Classic recipe with name "%s" count be found' % pn) - sys.exit(1) - - if options.dryrun: - raise DryRunRollbackException() - except DryRunRollbackException: - pass - - sys.exit(0) - - -if __name__ == "__main__": - main() diff --git a/layerindex/tools/import_wiki_layers.py b/layerindex/tools/import_wiki_layers.py deleted file mode 100755 index 71f26ea..0000000 --- a/layerindex/tools/import_wiki_layers.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -# Import layer index wiki page into database -# -# Copyright (C) 2013 Intel Corporation -# Author: Paul Eggleton -# -# Licensed under the MIT license, see COPYING.MIT for details - - -import sys -import os.path - -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) - -import optparse -import re -import utils - -logger = utils.logger_create('LayerIndexImport') - - -class DryRunRollbackException(Exception): - pass - - -def main(): - - parser = optparse.OptionParser( - usage = """ - %prog [options]""") - - options, args = parser.parse_args(sys.argv) - - utils.setup_django() - from layerindex.models import LayerItem, LayerBranch, LayerDependency - from django.db import transaction - - import httplib - conn = httplib.HTTPConnection("www.openembedded.org") - conn.request("GET", "/wiki/LayerIndex?action=raw") - resp = conn.getresponse() - if resp.status in [200, 302]: - data = resp.read() - in_table = False - layer_type = 'M' - nowiki_re = re.compile(r'') - link_re = re.compile(r'\[(http.*) +link\]') - readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') - master_branch = utils.get_branch('master') - core_layer = None - with transaction.atomic(): - for line in data.splitlines(): - if line.startswith('{|'): - in_table = True - continue - if in_table: - if line.startswith('|}'): - # We're done - break - elif line.startswith('!'): - section = line.split('|', 1)[1].strip("'") - if section.startswith('Base'): - layer_type = 'A' - elif section.startswith('Board'): - layer_type = 'B' - elif section.startswith('Software'): - layer_type = 'S' - elif section.startswith('Distribution'): - layer_type = 'D' - else: - layer_type = 'M' - elif not line.startswith('|-'): - if line.startswith("|| ''"): - continue - fields = line.split('||') - layer = LayerItem() - layer.name = fields[1].strip() - if ' ' in layer.name: - logger.warn('Skipping layer %s - name invalid' % layer.name) - continue - logger.info('Adding layer %s' % layer.name) - layer.status = 'P' - layer.layer_type = layer_type - layer.summary = fields[2].strip() - layer.description = layer.summary - if len(fields) > 6: - res = link_re.match(fields[6].strip()) - if res: - link = res.groups(1)[0].strip() - if link.endswith('/README') or readme_re.search(link): - link = 'README' - layer.usage_url = link - - repoval = nowiki_re.sub('', fields[4]).strip() - layer.vcs_url = repoval - if repoval.startswith('git://git.openembedded.org/'): - reponame = re.sub('^.*/', '', repoval) - layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame - layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%' - elif repoval.startswith('git://git.yoctoproject.org/'): - reponame = re.sub('^.*/', '', repoval) - layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame - layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' - layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%' - elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'): - reponame = re.sub('^.*github.com/', '', repoval) - reponame = re.sub('.git$', '', reponame) - layer.vcs_web_url = 'http://github.com/' + reponame - layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/' - layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/' - layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%' - elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'): - reponame = re.sub('^.*gitlab.com/', '', repoval) - reponame = re.sub('.git$', '', reponame) - layer.vcs_web_url = 'http://gitlab.com/' + reponame - layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/' - layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/' - layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%' - elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'): - reponame = re.sub('^.*bitbucket.org/', '', repoval) - reponame = re.sub('.git$', '', reponame) - layer.vcs_web_url = 'http://bitbucket.org/' + reponame - layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' - layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' - layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%' - elif '.git' in repoval: - res = link_re.match(fields[5].strip()) - layer.vcs_web_url = res.groups(1)[0] - layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url) - layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url) - layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url) - - layer.save() - layerbranch = LayerBranch() - layerbranch.layer = layer - layerbranch.branch = master_branch - layerbranch.vcs_subdir = fields[3].strip() - layerbranch.save() - if layer.name != 'openembedded-core': - if not core_layer: - core_layer = utils.get_layer('openembedded-core') - if core_layer: - layerdep = LayerDependency() - layerdep.layerbranch = layerbranch - layerdep.dependency = core_layer - layerdep.save() - else: - logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) - - sys.exit(0) - - -if __name__ == "__main__": - main()