MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-19 20:59:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

tools: fix for Django 1.8

Browse Source 
Fix the transaction handling code to work with Django 1.8 in most of the
tools scripts. (Some of these are no longer used, but still serve as
examples of how to import data and update the database.)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2017-01-16 11:35:57 +13:00
parent 072c7d6656
commit 45d307369f
4 changed files with 179 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
layerindex/tools/import_classic.py
View File

@ -27,6 +27,10 @@ import recipeparse
logger = utils.logger_create('LayerIndexUpdate') logger = utils.logger_create('LayerIndexUpdate')
class DryRunRollbackException(Exception):
pass
def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir): def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir):
fn = str(os.path.join(path, recipe.filename)) fn = str(os.path.join(path, recipe.filename))
try: try:
@ -154,55 +158,51 @@ def main():
# Clear the default value of HOMEPAGE ('unknown') # Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '') tinfoil.config_data.setVar('HOMEPAGE', '')
transaction.enter_transaction_management()
transaction.managed(True)
try: try:
layerdir_start = os.path.normpath(oeclassicpath) + os.sep with transaction.atomic():
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layermachines = Machine.objects.filter(layerbranch=layerbranch) layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layerdistros = Distro.objects.filter(layerbranch=layerbranch) layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerdistros = Distro.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try: try:
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch) config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
except recipeparse.RecipeParseError as e: except recipeparse.RecipeParseError as e:
logger.error(str(e)) logger.error(str(e))
transaction.rollback() sys.exit(1)
sys.exit(1)
layerrecipes.delete() layerrecipes.delete()
layermachines.delete() layermachines.delete()
layerdistros.delete() layerdistros.delete()
layerappends.delete() layerappends.delete()
layerclasses.delete() layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath): for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs: if '.git' in dirs:
dirs.remove('.git') dirs.remove('.git')
for f in files: for f in files:
fullpath = os.path.join(root, f) fullpath = os.path.join(root, f)
(typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
if typename == 'recipe': if typename == 'recipe':
recipe = ClassicRecipe() recipe = ClassicRecipe()
recipe.layerbranch = layerbranch recipe.layerbranch = layerbranch
recipe.filename = filename recipe.filename = filename
recipe.filepath = filepath recipe.filepath = filepath
update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath) update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath)
recipe.save() recipe.save()
layerbranch.vcs_last_fetch = datetime.now() layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save() layerbranch.save()
if options.dryrun: if options.dryrun:
transaction.rollback() raise DryRunRollbackException()
else: except DryRunRollbackException:
transaction.commit() pass
except: except:
import traceback import traceback
traceback.print_exc() traceback.print_exc()
transaction.rollback()
finally: finally:
transaction.leave_transaction_management()
tinfoil.shutdown() tinfoil.shutdown()
shutil.rmtree(tempdir) shutil.rmtree(tempdir)

176
layerindex/tools/import_classic_wiki.py
View File

@ -21,6 +21,10 @@ import logging
logger = utils.logger_create('LayerIndexImport') logger = utils.logger_create('LayerIndexImport')
class DryRunRollbackException(Exception):
pass
def read_page(site, path): def read_page(site, path):
ret = {} ret = {}
import httplib import httplib
@ -98,103 +102,97 @@ def main():
recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw") recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw")
recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw") recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw")
transaction.enter_transaction_management()
transaction.managed(True)
try: try:
recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) with transaction.atomic():
for pn, comment in recipes.items(): recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
newpn = '' for pn, comment in recipes.items():
newlayer = '' newpn = ''
status = 'U' newlayer = ''
comment = comment.strip(' -') status = 'U'
if 'provided by' in comment: comment = comment.strip(' -')
res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) if 'provided by' in comment:
if res: res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
newpn = res.group(1) if res:
newlayer = res.group(2) newpn = res.group(1)
comment = res.group(3) newlayer = res.group(2)
if pn.endswith('-native') or pn.endswith('-cross'): comment = res.group(3)
status = 'P' if pn.endswith('-native') or pn.endswith('-cross'):
else: status = 'P'
else:
status = 'R'
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
if not res:
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
if not res:
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
if res:
newpn = res.group(1)
newlayer = res.group(2)
comment = res.group(3)
status = 'R' status = 'R'
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: elif 'obsolete' in comment or 'superseded' in comment:
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
if not res: if res:
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) newpn = res.group(1)
if not res: newlayer = res.group(2)
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) comment = res.group(3)
if res: elif comment.startswith('superseded by'):
newpn = res.group(1) comment = comment[14:]
newlayer = res.group(2) elif comment.startswith('obsolete'):
comment = res.group(3) comment = comment[9:]
status = 'R' status = 'O'
elif 'obsolete' in comment or 'superseded' in comment: elif 'PACKAGECONFIG' in comment:
res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
if res: if res:
newpn = res.group(1) newpn = res.group(1)
newlayer = res.group(2) newlayer = res.group(2)
comment = res.group(3) comment = res.group(3)
elif comment.startswith('superseded by'): status = 'C'
comment = comment[14:]
elif comment.startswith('obsolete'):
comment = comment[9:]
status = 'O'
elif 'PACKAGECONFIG' in comment:
res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
if res:
newpn = res.group(1)
newlayer = res.group(2)
comment = res.group(3)
status = 'C'
if newlayer: if newlayer:
if newlayer.lower() == 'oe-core': if newlayer.lower() == 'oe-core':
newlayer = 'openembedded-core' newlayer = 'openembedded-core'
# Remove all links from comments because they'll be picked up as categories # Remove all links from comments because they'll be picked up as categories
comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
# Split out categories # Split out categories
categories = re.findall(r'\[([^]]*)\]', comment) categories = re.findall(r'\[([^]]*)\]', comment)
for cat in categories: for cat in categories:
comment = comment.replace('[%s]' % cat, '') comment = comment.replace('[%s]' % cat, '')
if '(GPE)' in comment or pn.startswith('gpe'): if '(GPE)' in comment or pn.startswith('gpe'):
categories.append('GPE') categories.append('GPE')
comment = comment.replace('(GPE)', '') comment = comment.replace('(GPE)', '')
comment = comment.strip('- ') comment = comment.strip('- ')
logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn) recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn)
if recipequery: if recipequery:
for recipe in recipequery: for recipe in recipequery:
recipe.cover_layerbranch = None recipe.cover_layerbranch = None
if newlayer: if newlayer:
res = list(LayerItem.objects.filter(name=newlayer)[:1]) res = list(LayerItem.objects.filter(name=newlayer)[:1])
if res: if res:
newlayeritem = res[0] newlayeritem = res[0]
recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
else: else:
logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
recipe.cover_pn = newpn recipe.cover_pn = newpn
recipe.cover_status = status recipe.cover_status = status
recipe.cover_verified = True recipe.cover_verified = True
recipe.cover_comment = comment recipe.cover_comment = comment
recipe.classic_category = " ".join(categories) recipe.classic_category = " ".join(categories)
recipe.save() recipe.save()
else: else:
logger.info('No OE-Classic recipe with name "%s" count be found' % pn) logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
sys.exit(1) sys.exit(1)
if options.dryrun: if options.dryrun:
transaction.rollback() raise DryRunRollbackException()
else: except DryRunRollbackException:
transaction.commit() pass
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
sys.exit(0) sys.exit(0)

14
layerindex/tools/import_wiki_layers.py
View File

@ -20,6 +20,10 @@ import utils
logger = utils.logger_create('LayerIndexImport') logger = utils.logger_create('LayerIndexImport')
class DryRunRollbackException(Exception):
pass
def main(): def main():
parser = optparse.OptionParser( parser = optparse.OptionParser(
@ -45,9 +49,7 @@ def main():
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master') master_branch = utils.get_branch('master')
core_layer = None core_layer = None
transaction.enter_transaction_management() with transaction.atomic():
transaction.managed(True)
try:
for line in data.splitlines(): for line in data.splitlines():
if line.startswith('{|'): if line.startswith('{|'):
in_table = True in_table = True
@ -140,12 +142,6 @@ def main():
layerdep.layerbranch = layerbranch layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer layerdep.dependency = core_layer
layerdep.save() layerdep.save()
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
else: else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))

97
layerindex/tools/update_classic_status.py
View File

@ -19,6 +19,9 @@ import logging
logger = utils.logger_create('LayerIndexClassicUpdate') logger = utils.logger_create('LayerIndexClassicUpdate')
class DryRunRollbackException(Exception):
pass
def main(): def main():
@ -62,61 +65,55 @@ def main():
logger.error("Specified branch %s does not exist in database" % options.branch) logger.error("Specified branch %s does not exist in database" % options.branch)
sys.exit(1) sys.exit(1)
transaction.enter_transaction_management()
transaction.managed(True)
try: try:
def recipe_pn_query(pn): with transaction.atomic():
return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference') def recipe_pn_query(pn):
return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference')
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N']) recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N'])
for recipe in recipequery: for recipe in recipequery:
replquery = recipe_pn_query(recipe.pn) replquery = recipe_pn_query(recipe.pn)
found = False found = False
for replrecipe in replquery: for replrecipe in replquery:
logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_status = 'D' recipe.cover_status = 'D'
recipe.cover_verified = False recipe.cover_verified = False
recipe.save() recipe.save()
found = True found = True
break break
if not found: if not found:
if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'):
searchpn, _, suffix = recipe.pn.rpartition('-') searchpn, _, suffix = recipe.pn.rpartition('-')
replquery = recipe_pn_query(searchpn) replquery = recipe_pn_query(searchpn)
for replrecipe in replquery:
if suffix in replrecipe.bbclassextend.split():
logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'P'
recipe.cover_verified = False
recipe.save()
found = True
break
if not found and recipe.pn.endswith('-nativesdk'):
searchpn, _, _ = recipe.pn.rpartition('-')
replquery = recipe_pn_query('nativesdk-%s' % searchpn)
for replrecipe in replquery: for replrecipe in replquery:
logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) if suffix in replrecipe.bbclassextend.split():
recipe.cover_layerbranch = replrecipe.layerbranch logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_pn = replrecipe.pn recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_status = 'R' recipe.cover_pn = replrecipe.pn
recipe.cover_verified = False recipe.cover_status = 'P'
recipe.save() recipe.cover_verified = False
found = True recipe.save()
break found = True
break
if not found and recipe.pn.endswith('-nativesdk'):
searchpn, _, _ = recipe.pn.rpartition('-')
replquery = recipe_pn_query('nativesdk-%s' % searchpn)
for replrecipe in replquery:
logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'R'
recipe.cover_verified = False
recipe.save()
found = True
break
if options.dryrun: if options.dryrun:
transaction.rollback() raise DryRunRollbackException()
else: except DryRunRollbackException:
transaction.commit() pass
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
sys.exit(0) sys.exit(0)