mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 12:19:02 +02:00
tools: fix for Django 1.8
Fix the transaction handling code to work with Django 1.8 in most of the tools scripts. (Some of these are no longer used, but still serve as examples of how to import data and update the database.) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton
parent
072c7d6656
commit
45d307369f
|
|
@ -27,6 +27,10 @@ import recipeparse
|
|||
logger = utils.logger_create('LayerIndexUpdate')
|
||||
|
||||
|
||||
class DryRunRollbackException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir):
|
||||
fn = str(os.path.join(path, recipe.filename))
|
||||
try:
|
||||
|
|
@ -154,55 +158,51 @@ def main():
|
|||
# Clear the default value of HOMEPAGE ('unknown')
|
||||
tinfoil.config_data.setVar('HOMEPAGE', '')
|
||||
|
||||
transaction.enter_transaction_management()
|
||||
transaction.managed(True)
|
||||
try:
|
||||
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
|
||||
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
|
||||
layermachines = Machine.objects.filter(layerbranch=layerbranch)
|
||||
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
|
||||
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
|
||||
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
|
||||
with transaction.atomic():
|
||||
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
|
||||
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
|
||||
layermachines = Machine.objects.filter(layerbranch=layerbranch)
|
||||
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
|
||||
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
|
||||
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
|
||||
|
||||
try:
|
||||
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
|
||||
except recipeparse.RecipeParseError as e:
|
||||
logger.error(str(e))
|
||||
transaction.rollback()
|
||||
sys.exit(1)
|
||||
try:
|
||||
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
|
||||
except recipeparse.RecipeParseError as e:
|
||||
logger.error(str(e))
|
||||
sys.exit(1)
|
||||
|
||||
layerrecipes.delete()
|
||||
layermachines.delete()
|
||||
layerdistros.delete()
|
||||
layerappends.delete()
|
||||
layerclasses.delete()
|
||||
for root, dirs, files in os.walk(oeclassicpath):
|
||||
if '.git' in dirs:
|
||||
dirs.remove('.git')
|
||||
for f in files:
|
||||
fullpath = os.path.join(root, f)
|
||||
(typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
|
||||
if typename == 'recipe':
|
||||
recipe = ClassicRecipe()
|
||||
recipe.layerbranch = layerbranch
|
||||
recipe.filename = filename
|
||||
recipe.filepath = filepath
|
||||
update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath)
|
||||
recipe.save()
|
||||
layerrecipes.delete()
|
||||
layermachines.delete()
|
||||
layerdistros.delete()
|
||||
layerappends.delete()
|
||||
layerclasses.delete()
|
||||
for root, dirs, files in os.walk(oeclassicpath):
|
||||
if '.git' in dirs:
|
||||
dirs.remove('.git')
|
||||
for f in files:
|
||||
fullpath = os.path.join(root, f)
|
||||
(typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
|
||||
if typename == 'recipe':
|
||||
recipe = ClassicRecipe()
|
||||
recipe.layerbranch = layerbranch
|
||||
recipe.filename = filename
|
||||
recipe.filepath = filepath
|
||||
update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath)
|
||||
recipe.save()
|
||||
|
||||
layerbranch.vcs_last_fetch = datetime.now()
|
||||
layerbranch.save()
|
||||
layerbranch.vcs_last_fetch = datetime.now()
|
||||
layerbranch.save()
|
||||
|
||||
if options.dryrun:
|
||||
transaction.rollback()
|
||||
else:
|
||||
transaction.commit()
|
||||
if options.dryrun:
|
||||
raise DryRunRollbackException()
|
||||
except DryRunRollbackException:
|
||||
pass
|
||||
except:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
transaction.rollback()
|
||||
finally:
|
||||
transaction.leave_transaction_management()
|
||||
tinfoil.shutdown()
|
||||
|
||||
shutil.rmtree(tempdir)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,10 @@ import logging
|
|||
logger = utils.logger_create('LayerIndexImport')
|
||||
|
||||
|
||||
class DryRunRollbackException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def read_page(site, path):
|
||||
ret = {}
|
||||
import httplib
|
||||
|
|
@ -98,103 +102,97 @@ def main():
|
|||
recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw")
|
||||
recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw")
|
||||
|
||||
transaction.enter_transaction_management()
|
||||
transaction.managed(True)
|
||||
try:
|
||||
recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
|
||||
for pn, comment in recipes.items():
|
||||
newpn = ''
|
||||
newlayer = ''
|
||||
status = 'U'
|
||||
comment = comment.strip(' -')
|
||||
if 'provided by' in comment:
|
||||
res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
if pn.endswith('-native') or pn.endswith('-cross'):
|
||||
status = 'P'
|
||||
else:
|
||||
with transaction.atomic():
|
||||
recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items()))
|
||||
for pn, comment in recipes.items():
|
||||
newpn = ''
|
||||
newlayer = ''
|
||||
status = 'U'
|
||||
comment = comment.strip(' -')
|
||||
if 'provided by' in comment:
|
||||
res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
if pn.endswith('-native') or pn.endswith('-cross'):
|
||||
status = 'P'
|
||||
else:
|
||||
status = 'R'
|
||||
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
|
||||
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if not res:
|
||||
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if not res:
|
||||
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
status = 'R'
|
||||
elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment:
|
||||
res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if not res:
|
||||
res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if not res:
|
||||
res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
status = 'R'
|
||||
elif 'obsolete' in comment or 'superseded' in comment:
|
||||
res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
elif comment.startswith('superseded by'):
|
||||
comment = comment[14:]
|
||||
elif comment.startswith('obsolete'):
|
||||
comment = comment[9:]
|
||||
status = 'O'
|
||||
elif 'PACKAGECONFIG' in comment:
|
||||
res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
status = 'C'
|
||||
elif 'obsolete' in comment or 'superseded' in comment:
|
||||
res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
elif comment.startswith('superseded by'):
|
||||
comment = comment[14:]
|
||||
elif comment.startswith('obsolete'):
|
||||
comment = comment[9:]
|
||||
status = 'O'
|
||||
elif 'PACKAGECONFIG' in comment:
|
||||
res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment)
|
||||
if res:
|
||||
newpn = res.group(1)
|
||||
newlayer = res.group(2)
|
||||
comment = res.group(3)
|
||||
status = 'C'
|
||||
|
||||
if newlayer:
|
||||
if newlayer.lower() == 'oe-core':
|
||||
newlayer = 'openembedded-core'
|
||||
if newlayer:
|
||||
if newlayer.lower() == 'oe-core':
|
||||
newlayer = 'openembedded-core'
|
||||
|
||||
# Remove all links from comments because they'll be picked up as categories
|
||||
comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
|
||||
# Split out categories
|
||||
categories = re.findall(r'\[([^]]*)\]', comment)
|
||||
for cat in categories:
|
||||
comment = comment.replace('[%s]' % cat, '')
|
||||
if '(GPE)' in comment or pn.startswith('gpe'):
|
||||
categories.append('GPE')
|
||||
comment = comment.replace('(GPE)', '')
|
||||
# Remove all links from comments because they'll be picked up as categories
|
||||
comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment)
|
||||
# Split out categories
|
||||
categories = re.findall(r'\[([^]]*)\]', comment)
|
||||
for cat in categories:
|
||||
comment = comment.replace('[%s]' % cat, '')
|
||||
if '(GPE)' in comment or pn.startswith('gpe'):
|
||||
categories.append('GPE')
|
||||
comment = comment.replace('(GPE)', '')
|
||||
|
||||
comment = comment.strip('- ')
|
||||
comment = comment.strip('- ')
|
||||
|
||||
logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))
|
||||
logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment))
|
||||
|
||||
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn)
|
||||
if recipequery:
|
||||
for recipe in recipequery:
|
||||
recipe.cover_layerbranch = None
|
||||
if newlayer:
|
||||
res = list(LayerItem.objects.filter(name=newlayer)[:1])
|
||||
if res:
|
||||
newlayeritem = res[0]
|
||||
recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
|
||||
else:
|
||||
logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
|
||||
recipe.cover_pn = newpn
|
||||
recipe.cover_status = status
|
||||
recipe.cover_verified = True
|
||||
recipe.cover_comment = comment
|
||||
recipe.classic_category = " ".join(categories)
|
||||
recipe.save()
|
||||
else:
|
||||
logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
|
||||
sys.exit(1)
|
||||
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn)
|
||||
if recipequery:
|
||||
for recipe in recipequery:
|
||||
recipe.cover_layerbranch = None
|
||||
if newlayer:
|
||||
res = list(LayerItem.objects.filter(name=newlayer)[:1])
|
||||
if res:
|
||||
newlayeritem = res[0]
|
||||
recipe.cover_layerbranch = newlayeritem.get_layerbranch('master')
|
||||
else:
|
||||
logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn))
|
||||
recipe.cover_pn = newpn
|
||||
recipe.cover_status = status
|
||||
recipe.cover_verified = True
|
||||
recipe.cover_comment = comment
|
||||
recipe.classic_category = " ".join(categories)
|
||||
recipe.save()
|
||||
else:
|
||||
logger.info('No OE-Classic recipe with name "%s" count be found' % pn)
|
||||
sys.exit(1)
|
||||
|
||||
if options.dryrun:
|
||||
transaction.rollback()
|
||||
else:
|
||||
transaction.commit()
|
||||
except:
|
||||
transaction.rollback()
|
||||
raise
|
||||
finally:
|
||||
transaction.leave_transaction_management()
|
||||
if options.dryrun:
|
||||
raise DryRunRollbackException()
|
||||
except DryRunRollbackException:
|
||||
pass
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ import utils
|
|||
logger = utils.logger_create('LayerIndexImport')
|
||||
|
||||
|
||||
class DryRunRollbackException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
parser = optparse.OptionParser(
|
||||
|
|
@ -45,9 +49,7 @@ def main():
|
|||
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
|
||||
master_branch = utils.get_branch('master')
|
||||
core_layer = None
|
||||
transaction.enter_transaction_management()
|
||||
transaction.managed(True)
|
||||
try:
|
||||
with transaction.atomic():
|
||||
for line in data.splitlines():
|
||||
if line.startswith('{|'):
|
||||
in_table = True
|
||||
|
|
@ -140,12 +142,6 @@ def main():
|
|||
layerdep.layerbranch = layerbranch
|
||||
layerdep.dependency = core_layer
|
||||
layerdep.save()
|
||||
transaction.commit()
|
||||
except:
|
||||
transaction.rollback()
|
||||
raise
|
||||
finally:
|
||||
transaction.leave_transaction_management()
|
||||
else:
|
||||
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ import logging
|
|||
|
||||
logger = utils.logger_create('LayerIndexClassicUpdate')
|
||||
|
||||
class DryRunRollbackException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
|
|
@ -62,61 +65,55 @@ def main():
|
|||
logger.error("Specified branch %s does not exist in database" % options.branch)
|
||||
sys.exit(1)
|
||||
|
||||
transaction.enter_transaction_management()
|
||||
transaction.managed(True)
|
||||
try:
|
||||
def recipe_pn_query(pn):
|
||||
return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference')
|
||||
with transaction.atomic():
|
||||
def recipe_pn_query(pn):
|
||||
return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference')
|
||||
|
||||
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N'])
|
||||
for recipe in recipequery:
|
||||
replquery = recipe_pn_query(recipe.pn)
|
||||
found = False
|
||||
for replrecipe in replquery:
|
||||
logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name))
|
||||
recipe.cover_layerbranch = replrecipe.layerbranch
|
||||
recipe.cover_status = 'D'
|
||||
recipe.cover_verified = False
|
||||
recipe.save()
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'):
|
||||
searchpn, _, suffix = recipe.pn.rpartition('-')
|
||||
replquery = recipe_pn_query(searchpn)
|
||||
for replrecipe in replquery:
|
||||
if suffix in replrecipe.bbclassextend.split():
|
||||
logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
|
||||
recipe.cover_layerbranch = replrecipe.layerbranch
|
||||
recipe.cover_pn = replrecipe.pn
|
||||
recipe.cover_status = 'P'
|
||||
recipe.cover_verified = False
|
||||
recipe.save()
|
||||
found = True
|
||||
break
|
||||
if not found and recipe.pn.endswith('-nativesdk'):
|
||||
searchpn, _, _ = recipe.pn.rpartition('-')
|
||||
replquery = recipe_pn_query('nativesdk-%s' % searchpn)
|
||||
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N'])
|
||||
for recipe in recipequery:
|
||||
replquery = recipe_pn_query(recipe.pn)
|
||||
found = False
|
||||
for replrecipe in replquery:
|
||||
logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name))
|
||||
recipe.cover_layerbranch = replrecipe.layerbranch
|
||||
recipe.cover_status = 'D'
|
||||
recipe.cover_verified = False
|
||||
recipe.save()
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'):
|
||||
searchpn, _, suffix = recipe.pn.rpartition('-')
|
||||
replquery = recipe_pn_query(searchpn)
|
||||
for replrecipe in replquery:
|
||||
logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
|
||||
recipe.cover_layerbranch = replrecipe.layerbranch
|
||||
recipe.cover_pn = replrecipe.pn
|
||||
recipe.cover_status = 'R'
|
||||
recipe.cover_verified = False
|
||||
recipe.save()
|
||||
found = True
|
||||
break
|
||||
if suffix in replrecipe.bbclassextend.split():
|
||||
logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
|
||||
recipe.cover_layerbranch = replrecipe.layerbranch
|
||||
recipe.cover_pn = replrecipe.pn
|
||||
recipe.cover_status = 'P'
|
||||
recipe.cover_verified = False
|
||||
recipe.save()
|
||||
found = True
|
||||
break
|
||||
if not found and recipe.pn.endswith('-nativesdk'):
|
||||
searchpn, _, _ = recipe.pn.rpartition('-')
|
||||
replquery = recipe_pn_query('nativesdk-%s' % searchpn)
|
||||
for replrecipe in replquery:
|
||||
logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
|
||||
recipe.cover_layerbranch = replrecipe.layerbranch
|
||||
recipe.cover_pn = replrecipe.pn
|
||||
recipe.cover_status = 'R'
|
||||
recipe.cover_verified = False
|
||||
recipe.save()
|
||||
found = True
|
||||
break
|
||||
|
||||
|
||||
if options.dryrun:
|
||||
transaction.rollback()
|
||||
else:
|
||||
transaction.commit()
|
||||
except:
|
||||
transaction.rollback()
|
||||
raise
|
||||
finally:
|
||||
transaction.leave_transaction_management()
|
||||
if options.dryrun:
|
||||
raise DryRunRollbackException()
|
||||
except DryRunRollbackException:
|
||||
pass
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user