mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 20:59:01 +02:00
Drop wiki import scripts
These scripts have not been used or maintained for some time; I only really kept them around in case it was useful as an example of how to poke data into the database, but we have plenty of those in other scripts. Signed-off-by: Paul Eggleton <bluelightning@bluelightning.org>
This commit is contained in:
Paul Eggleton
parent
9cf0829f41
commit
1327fe1b42
|
|
@ -1,201 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Import migration info from OE-Classic recipes wiki page into OE
|
|
||||||
# layer index database
|
|
||||||
#
|
|
||||||
# Copyright (C) 2013 Intel Corporation
|
|
||||||
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
||||||
#
|
|
||||||
# Licensed under the MIT license, see COPYING.MIT for details
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os.path
|
|
||||||
|
|
||||||
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
|
|
||||||
|
|
||||||
import optparse
|
|
||||||
import re
|
|
||||||
import utils
|
|
||||||
import logging
|
|
||||||
|
|
||||||
logger = utils.logger_create('LayerIndexImport')
|
|
||||||
|
|
||||||
|
|
||||||
class DryRunRollbackException(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def read_page(site, path):
|
|
||||||
ret = {}
|
|
||||||
import httplib
|
|
||||||
conn = httplib.HTTPConnection(site)
|
|
||||||
conn.request("GET", path)
|
|
||||||
resp = conn.getresponse()
|
|
||||||
if resp.status in [200, 302]:
|
|
||||||
data = resp.read()
|
|
||||||
in_table = False
|
|
||||||
for line in data.splitlines():
|
|
||||||
if line.startswith('{|'):
|
|
||||||
in_table = True
|
|
||||||
continue
|
|
||||||
if in_table:
|
|
||||||
if line.startswith('|}'):
|
|
||||||
# We're done
|
|
||||||
in_table = False
|
|
||||||
continue
|
|
||||||
elif line.startswith('!'):
|
|
||||||
pass
|
|
||||||
elif not line.startswith('|-'):
|
|
||||||
if line.startswith("|| ''"):
|
|
||||||
continue
|
|
||||||
fields = line.split('||')
|
|
||||||
pn = fields[0].strip('|[]').split()[1]
|
|
||||||
comment = fields[1]
|
|
||||||
if comment:
|
|
||||||
ret[pn] = comment
|
|
||||||
else:
|
|
||||||
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def main():
|
|
||||||
|
|
||||||
parser = optparse.OptionParser(
|
|
||||||
usage = """
|
|
||||||
%prog [options]""")
|
|
||||||
|
|
||||||
parser.add_option("-b", "--branch",
|
|
||||||
help = "Specify branch to import into",
|
|
||||||
action="store", dest="branch", default='oe-classic')
|
|
||||||
parser.add_option("-l", "--layer",
|
|
||||||
help = "Specify layer to import into",
|
|
||||||
action="store", dest="layer", default='oe-classic')
|
|
||||||
parser.add_option("-n", "--dry-run",
|
|
||||||
help = "Don't write any data back to the database",
|
|
||||||
action="store_true", dest="dryrun")
|
|
||||||
parser.add_option("-d", "--debug",
|
|
||||||
help = "Enable debug output",
|
|
||||||
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
||||||
parser.add_option("-q", "--quiet",
|
|
||||||
help = "Hide all output except error messages",
|
|
||||||
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
||||||
|
|
||||||
options, args = parser.parse_args(sys.argv)
|
|
||||||
|
|
||||||
utils.setup_django()
|
|
||||||
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe
|
|
||||||
from django.db import transaction
|
|
||||||
|
|
||||||
logger.setLevel(options.loglevel)
|
|
||||||
|
|
||||||
res = list(LayerItem.objects.filter(name=options.layer)[:1])
|
|
||||||
if res:
|
|
||||||
layer = res[0]
|
|
||||||
else:
|
|
||||||
logger.error('Specified layer %s does not exist in database' % options.layer)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
layerbranch = layer.get_layerbranch(options.branch)
|
|
||||||
if not layerbranch:
|
|
||||||
logger.error("Specified branch %s does not exist in database" % options.branch)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw")
|
|
||||||
recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw")
|
|
||||||
|
|
||||||
try:
|
|
||||||
with transaction.atomic():
|
|
||||||
recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
|
|
||||||
for pn, comment in recipes.items():
|
|
||||||
newpn = ''
|
|
||||||
newlayer = ''
|
|
||||||
status = 'U'
|
|
||||||
comment = comment.strip(' -')
|
|
||||||
if 'provided by' in comment:
|
|
||||||
res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
||||||
if res:
|
|
||||||
newpn = res.group(1)
|
|
||||||
newlayer = res.group(2)
|
|
||||||
comment = res.group(3)
|
|
||||||
if pn.endswith('-native') or pn.endswith('-cross'):
|
|
||||||
status = 'P'
|
|
||||||
else:
|
|
||||||
status = 'R'
|
|
||||||
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
|
|
||||||
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
||||||
if not res:
|
|
||||||
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
||||||
if not res:
|
|
||||||
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
||||||
if res:
|
|
||||||
newpn = res.group(1)
|
|
||||||
newlayer = res.group(2)
|
|
||||||
comment = res.group(3)
|
|
||||||
status = 'R'
|
|
||||||
elif 'obsolete' in comment or 'superseded' in comment:
|
|
||||||
res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
||||||
if res:
|
|
||||||
newpn = res.group(1)
|
|
||||||
newlayer = res.group(2)
|
|
||||||
comment = res.group(3)
|
|
||||||
elif comment.startswith('superseded by'):
|
|
||||||
comment = comment[14:]
|
|
||||||
elif comment.startswith('obsolete'):
|
|
||||||
comment = comment[9:]
|
|
||||||
status = 'O'
|
|
||||||
elif 'PACKAGECONFIG' in comment:
|
|
||||||
res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
|
||||||
if res:
|
|
||||||
newpn = res.group(1)
|
|
||||||
newlayer = res.group(2)
|
|
||||||
comment = res.group(3)
|
|
||||||
status = 'C'
|
|
||||||
|
|
||||||
if newlayer:
|
|
||||||
if newlayer.lower() == 'oe-core':
|
|
||||||
newlayer = 'openembedded-core'
|
|
||||||
|
|
||||||
# Remove all links from comments because they'll be picked up as categories
|
|
||||||
comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
|
|
||||||
# Split out categories
|
|
||||||
categories = re.findall(r'\[([^]]*)\]', comment)
|
|
||||||
for cat in categories:
|
|
||||||
comment = comment.replace('[%s]' % cat, '')
|
|
||||||
if '(GPE)' in comment or pn.startswith('gpe'):
|
|
||||||
categories.append('GPE')
|
|
||||||
comment = comment.replace('(GPE)', '')
|
|
||||||
|
|
||||||
comment = comment.strip('- ')
|
|
||||||
|
|
||||||
logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))
|
|
||||||
|
|
||||||
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn).filter(deleted=False)
|
|
||||||
if recipequery:
|
|
||||||
for recipe in recipequery:
|
|
||||||
recipe.cover_layerbranch = None
|
|
||||||
if newlayer:
|
|
||||||
res = list(LayerItem.objects.filter(name=newlayer)[:1])
|
|
||||||
if res:
|
|
||||||
newlayeritem = res[0]
|
|
||||||
recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
|
|
||||||
else:
|
|
||||||
logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
|
|
||||||
recipe.cover_pn = newpn
|
|
||||||
recipe.cover_status = status
|
|
||||||
recipe.cover_verified = True
|
|
||||||
recipe.cover_comment = comment
|
|
||||||
recipe.classic_category = " ".join(categories)
|
|
||||||
recipe.save()
|
|
||||||
else:
|
|
||||||
logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if options.dryrun:
|
|
||||||
raise DryRunRollbackException()
|
|
||||||
except DryRunRollbackException:
|
|
||||||
pass
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
@ -1,158 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
# Import layer index wiki page into database
|
|
||||||
#
|
|
||||||
# Copyright (C) 2013 Intel Corporation
|
|
||||||
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
||||||
#
|
|
||||||
# Licensed under the MIT license, see COPYING.MIT for details
|
|
||||||
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import os.path
|
|
||||||
|
|
||||||
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
|
|
||||||
|
|
||||||
import optparse
|
|
||||||
import re
|
|
||||||
import utils
|
|
||||||
|
|
||||||
logger = utils.logger_create('LayerIndexImport')
|
|
||||||
|
|
||||||
|
|
||||||
class DryRunRollbackException(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
|
|
||||||
parser = optparse.OptionParser(
|
|
||||||
usage = """
|
|
||||||
%prog [options]""")
|
|
||||||
|
|
||||||
options, args = parser.parse_args(sys.argv)
|
|
||||||
|
|
||||||
utils.setup_django()
|
|
||||||
from layerindex.models import LayerItem, LayerBranch, LayerDependency
|
|
||||||
from django.db import transaction
|
|
||||||
|
|
||||||
import httplib
|
|
||||||
conn = httplib.HTTPConnection("www.openembedded.org")
|
|
||||||
conn.request("GET", "/wiki/LayerIndex?action=raw")
|
|
||||||
resp = conn.getresponse()
|
|
||||||
if resp.status in [200, 302]:
|
|
||||||
data = resp.read()
|
|
||||||
in_table = False
|
|
||||||
layer_type = 'M'
|
|
||||||
nowiki_re = re.compile(r'</?nowiki>')
|
|
||||||
link_re = re.compile(r'\[(http.*) +link\]')
|
|
||||||
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
|
|
||||||
master_branch = utils.get_branch('master')
|
|
||||||
core_layer = None
|
|
||||||
with transaction.atomic():
|
|
||||||
for line in data.splitlines():
|
|
||||||
if line.startswith('{|'):
|
|
||||||
in_table = True
|
|
||||||
continue
|
|
||||||
if in_table:
|
|
||||||
if line.startswith('|}'):
|
|
||||||
# We're done
|
|
||||||
break
|
|
||||||
elif line.startswith('!'):
|
|
||||||
section = line.split('|', 1)[1].strip("'")
|
|
||||||
if section.startswith('Base'):
|
|
||||||
layer_type = 'A'
|
|
||||||
elif section.startswith('Board'):
|
|
||||||
layer_type = 'B'
|
|
||||||
elif section.startswith('Software'):
|
|
||||||
layer_type = 'S'
|
|
||||||
elif section.startswith('Distribution'):
|
|
||||||
layer_type = 'D'
|
|
||||||
else:
|
|
||||||
layer_type = 'M'
|
|
||||||
elif not line.startswith('|-'):
|
|
||||||
if line.startswith("|| ''"):
|
|
||||||
continue
|
|
||||||
fields = line.split('||')
|
|
||||||
layer = LayerItem()
|
|
||||||
layer.name = fields[1].strip()
|
|
||||||
if ' ' in layer.name:
|
|
||||||
logger.warn('Skipping layer %s - name invalid' % layer.name)
|
|
||||||
continue
|
|
||||||
logger.info('Adding layer %s' % layer.name)
|
|
||||||
layer.status = 'P'
|
|
||||||
layer.layer_type = layer_type
|
|
||||||
layer.summary = fields[2].strip()
|
|
||||||
layer.description = layer.summary
|
|
||||||
if len(fields) > 6:
|
|
||||||
res = link_re.match(fields[6].strip())
|
|
||||||
if res:
|
|
||||||
link = res.groups(1)[0].strip()
|
|
||||||
if link.endswith('/README') or readme_re.search(link):
|
|
||||||
link = 'README'
|
|
||||||
layer.usage_url = link
|
|
||||||
|
|
||||||
repoval = nowiki_re.sub('', fields[4]).strip()
|
|
||||||
layer.vcs_url = repoval
|
|
||||||
if repoval.startswith('git://git.openembedded.org/'):
|
|
||||||
reponame = re.sub('^.*/', '', repoval)
|
|
||||||
layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame
|
|
||||||
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
|
|
||||||
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
|
|
||||||
layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%'
|
|
||||||
elif repoval.startswith('git://git.yoctoproject.org/'):
|
|
||||||
reponame = re.sub('^.*/', '', repoval)
|
|
||||||
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
|
|
||||||
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
||||||
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
||||||
layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%'
|
|
||||||
elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'):
|
|
||||||
reponame = re.sub('^.*github.com/', '', repoval)
|
|
||||||
reponame = re.sub('.git$', '', reponame)
|
|
||||||
layer.vcs_web_url = 'http://github.com/' + reponame
|
|
||||||
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
|
|
||||||
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
|
|
||||||
layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%'
|
|
||||||
elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'):
|
|
||||||
reponame = re.sub('^.*gitlab.com/', '', repoval)
|
|
||||||
reponame = re.sub('.git$', '', reponame)
|
|
||||||
layer.vcs_web_url = 'http://gitlab.com/' + reponame
|
|
||||||
layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/'
|
|
||||||
layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/'
|
|
||||||
layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%'
|
|
||||||
elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'):
|
|
||||||
reponame = re.sub('^.*bitbucket.org/', '', repoval)
|
|
||||||
reponame = re.sub('.git$', '', reponame)
|
|
||||||
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
|
|
||||||
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
|
|
||||||
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
|
|
||||||
layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%'
|
|
||||||
elif '.git' in repoval:
|
|
||||||
res = link_re.match(fields[5].strip())
|
|
||||||
layer.vcs_web_url = res.groups(1)[0]
|
|
||||||
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
|
|
||||||
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
|
|
||||||
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url)
|
|
||||||
|
|
||||||
layer.save()
|
|
||||||
layerbranch = LayerBranch()
|
|
||||||
layerbranch.layer = layer
|
|
||||||
layerbranch.branch = master_branch
|
|
||||||
layerbranch.vcs_subdir = fields[3].strip()
|
|
||||||
layerbranch.save()
|
|
||||||
if layer.name != 'openembedded-core':
|
|
||||||
if not core_layer:
|
|
||||||
core_layer = utils.get_layer('openembedded-core')
|
|
||||||
if core_layer:
|
|
||||||
layerdep = LayerDependency()
|
|
||||||
layerdep.layerbranch = layerbranch
|
|
||||||
layerdep.dependency = core_layer
|
|
||||||
layerdep.save()
|
|
||||||
else:
|
|
||||||
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
Loading…
Reference in New Issue
Block a user