From 45d307369fb8a7c833b601283a37d668b74a2bc1 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Mon, 16 Jan 2017 11:35:57 +1300 Subject: [PATCH] tools: fix for Django 1.8 Fix the transaction handling code to work with Django 1.8 in most of the tools scripts. (Some of these are no longer used, but still serve as examples of how to import data and update the database.) Signed-off-by: Paul Eggleton --- layerindex/tools/import_classic.py | 80 +++++----- layerindex/tools/import_classic_wiki.py | 176 +++++++++++----------- layerindex/tools/import_wiki_layers.py | 14 +- layerindex/tools/update_classic_status.py | 97 ++++++------ 4 files changed, 179 insertions(+), 188 deletions(-) diff --git a/layerindex/tools/import_classic.py b/layerindex/tools/import_classic.py index 61f5f90..c006e24 100755 --- a/layerindex/tools/import_classic.py +++ b/layerindex/tools/import_classic.py @@ -27,6 +27,10 @@ import recipeparse logger = utils.logger_create('LayerIndexUpdate') +class DryRunRollbackException(Exception): + pass + + def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir): fn = str(os.path.join(path, recipe.filename)) try: @@ -154,55 +158,51 @@ def main(): # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') - transaction.enter_transaction_management() - transaction.managed(True) try: - layerdir_start = os.path.normpath(oeclassicpath) + os.sep - layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) - layermachines = Machine.objects.filter(layerbranch=layerbranch) - layerdistros = Distro.objects.filter(layerbranch=layerbranch) - layerappends = BBAppend.objects.filter(layerbranch=layerbranch) - layerclasses = BBClass.objects.filter(layerbranch=layerbranch) + with transaction.atomic(): + layerdir_start = os.path.normpath(oeclassicpath) + os.sep + layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) + layermachines = Machine.objects.filter(layerbranch=layerbranch) + layerdistros = Distro.objects.filter(layerbranch=layerbranch) + layerappends = BBAppend.objects.filter(layerbranch=layerbranch) + layerclasses = BBClass.objects.filter(layerbranch=layerbranch) - try: - config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch) - except recipeparse.RecipeParseError as e: - logger.error(str(e)) - transaction.rollback() - sys.exit(1) + try: + config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch) + except recipeparse.RecipeParseError as e: + logger.error(str(e)) + sys.exit(1) - layerrecipes.delete() - layermachines.delete() - layerdistros.delete() - layerappends.delete() - layerclasses.delete() - for root, dirs, files in os.walk(oeclassicpath): - if '.git' in dirs: - dirs.remove('.git') - for f in files: - fullpath = os.path.join(root, f) - (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) - if typename == 'recipe': - recipe = ClassicRecipe() - recipe.layerbranch = layerbranch - recipe.filename = filename - recipe.filepath = filepath - update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath) - recipe.save() + layerrecipes.delete() + layermachines.delete() + layerdistros.delete() + layerappends.delete() + layerclasses.delete() + for root, dirs, files in os.walk(oeclassicpath): + if '.git' in dirs: + dirs.remove('.git') + for f in files: + fullpath = os.path.join(root, f) + (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) + if typename == 'recipe': + recipe = ClassicRecipe() + recipe.layerbranch = layerbranch + recipe.filename = filename + recipe.filepath = filepath + update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath) + recipe.save() - layerbranch.vcs_last_fetch = datetime.now() - layerbranch.save() + layerbranch.vcs_last_fetch = datetime.now() + layerbranch.save() - if options.dryrun: - transaction.rollback() - else: - transaction.commit() + if options.dryrun: + raise DryRunRollbackException() + except DryRunRollbackException: + pass except: import traceback traceback.print_exc() - transaction.rollback() finally: - transaction.leave_transaction_management() tinfoil.shutdown() shutil.rmtree(tempdir) diff --git a/layerindex/tools/import_classic_wiki.py b/layerindex/tools/import_classic_wiki.py index 48c71e9..fb516f1 100755 --- a/layerindex/tools/import_classic_wiki.py +++ b/layerindex/tools/import_classic_wiki.py @@ -21,6 +21,10 @@ import logging logger = utils.logger_create('LayerIndexImport') +class DryRunRollbackException(Exception): + pass + + def read_page(site, path): ret = {} import httplib @@ -98,103 +102,97 @@ def main(): recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw") recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw") - transaction.enter_transaction_management() - transaction.managed(True) try: - recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) - for pn, comment in recipes.items(): - newpn = '' - newlayer = '' - status = 'U' - comment = comment.strip(' -') - if 'provided by' in comment: - res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - if pn.endswith('-native') or pn.endswith('-cross'): - status = 'P' - else: + with transaction.atomic(): + recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) + for pn, comment in recipes.items(): + newpn = '' + newlayer = '' + status = 'U' + comment = comment.strip(' -') + if 'provided by' in comment: + res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) + if pn.endswith('-native') or pn.endswith('-cross'): + status = 'P' + else: + status = 'R' + elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: + res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if not res: + res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if not res: + res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) status = 'R' - elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: - res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'R' - elif 'obsolete' in comment or 'superseded' in comment: - res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - elif comment.startswith('superseded by'): - comment = comment[14:] - elif comment.startswith('obsolete'): - comment = comment[9:] - status = 'O' - elif 'PACKAGECONFIG' in comment: - res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'C' + elif 'obsolete' in comment or 'superseded' in comment: + res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) + elif comment.startswith('superseded by'): + comment = comment[14:] + elif comment.startswith('obsolete'): + comment = comment[9:] + status = 'O' + elif 'PACKAGECONFIG' in comment: + res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) + status = 'C' - if newlayer: - if newlayer.lower() == 'oe-core': - newlayer = 'openembedded-core' + if newlayer: + if newlayer.lower() == 'oe-core': + newlayer = 'openembedded-core' - # Remove all links from comments because they'll be picked up as categories - comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) - # Split out categories - categories = re.findall(r'\[([^]]*)\]', comment) - for cat in categories: - comment = comment.replace('[%s]' % cat, '') - if '(GPE)' in comment or pn.startswith('gpe'): - categories.append('GPE') - comment = comment.replace('(GPE)', '') + # Remove all links from comments because they'll be picked up as categories + comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) + # Split out categories + categories = re.findall(r'\[([^]]*)\]', comment) + for cat in categories: + comment = comment.replace('[%s]' % cat, '') + if '(GPE)' in comment or pn.startswith('gpe'): + categories.append('GPE') + comment = comment.replace('(GPE)', '') - comment = comment.strip('- ') + comment = comment.strip('- ') - logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) + logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn) - if recipequery: - for recipe in recipequery: - recipe.cover_layerbranch = None - if newlayer: - res = list(LayerItem.objects.filter(name=newlayer)[:1]) - if res: - newlayeritem = res[0] - recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') - else: - logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) - recipe.cover_pn = newpn - recipe.cover_status = status - recipe.cover_verified = True - recipe.cover_comment = comment - recipe.classic_category = " ".join(categories) - recipe.save() - else: - logger.info('No OE-Classic recipe with name "%s" count be found' % pn) - sys.exit(1) + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn) + if recipequery: + for recipe in recipequery: + recipe.cover_layerbranch = None + if newlayer: + res = list(LayerItem.objects.filter(name=newlayer)[:1]) + if res: + newlayeritem = res[0] + recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') + else: + logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) + recipe.cover_pn = newpn + recipe.cover_status = status + recipe.cover_verified = True + recipe.cover_comment = comment + recipe.classic_category = " ".join(categories) + recipe.save() + else: + logger.info('No OE-Classic recipe with name "%s" count be found' % pn) + sys.exit(1) - if options.dryrun: - transaction.rollback() - else: - transaction.commit() - except: - transaction.rollback() - raise - finally: - transaction.leave_transaction_management() + if options.dryrun: + raise DryRunRollbackException() + except DryRunRollbackException: + pass sys.exit(0) diff --git a/layerindex/tools/import_wiki_layers.py b/layerindex/tools/import_wiki_layers.py index f9f9457..bcf57f3 100755 --- a/layerindex/tools/import_wiki_layers.py +++ b/layerindex/tools/import_wiki_layers.py @@ -20,6 +20,10 @@ import utils logger = utils.logger_create('LayerIndexImport') +class DryRunRollbackException(Exception): + pass + + def main(): parser = optparse.OptionParser( @@ -45,9 +49,7 @@ def main(): readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') master_branch = utils.get_branch('master') core_layer = None - transaction.enter_transaction_management() - transaction.managed(True) - try: + with transaction.atomic(): for line in data.splitlines(): if line.startswith('{|'): in_table = True @@ -140,12 +142,6 @@ def main(): layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() - transaction.commit() - except: - transaction.rollback() - raise - finally: - transaction.leave_transaction_management() else: logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) diff --git a/layerindex/tools/update_classic_status.py b/layerindex/tools/update_classic_status.py index ade2461..f980e64 100755 --- a/layerindex/tools/update_classic_status.py +++ b/layerindex/tools/update_classic_status.py @@ -19,6 +19,9 @@ import logging logger = utils.logger_create('LayerIndexClassicUpdate') +class DryRunRollbackException(Exception): + pass + def main(): @@ -62,61 +65,55 @@ def main(): logger.error("Specified branch %s does not exist in database" % options.branch) sys.exit(1) - transaction.enter_transaction_management() - transaction.managed(True) try: - def recipe_pn_query(pn): - return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference') + with transaction.atomic(): + def recipe_pn_query(pn): + return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference') - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N']) - for recipe in recipequery: - replquery = recipe_pn_query(recipe.pn) - found = False - for replrecipe in replquery: - logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_status = 'D' - recipe.cover_verified = False - recipe.save() - found = True - break - if not found: - if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): - searchpn, _, suffix = recipe.pn.rpartition('-') - replquery = recipe_pn_query(searchpn) - for replrecipe in replquery: - if suffix in replrecipe.bbclassextend.split(): - logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'P' - recipe.cover_verified = False - recipe.save() - found = True - break - if not found and recipe.pn.endswith('-nativesdk'): - searchpn, _, _ = recipe.pn.rpartition('-') - replquery = recipe_pn_query('nativesdk-%s' % searchpn) + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N']) + for recipe in recipequery: + replquery = recipe_pn_query(recipe.pn) + found = False + for replrecipe in replquery: + logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_status = 'D' + recipe.cover_verified = False + recipe.save() + found = True + break + if not found: + if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): + searchpn, _, suffix = recipe.pn.rpartition('-') + replquery = recipe_pn_query(searchpn) for replrecipe in replquery: - logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'R' - recipe.cover_verified = False - recipe.save() - found = True - break + if suffix in replrecipe.bbclassextend.split(): + logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'P' + recipe.cover_verified = False + recipe.save() + found = True + break + if not found and recipe.pn.endswith('-nativesdk'): + searchpn, _, _ = recipe.pn.rpartition('-') + replquery = recipe_pn_query('nativesdk-%s' % searchpn) + for replrecipe in replquery: + logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'R' + recipe.cover_verified = False + recipe.save() + found = True + break - if options.dryrun: - transaction.rollback() - else: - transaction.commit() - except: - transaction.rollback() - raise - finally: - transaction.leave_transaction_management() + if options.dryrun: + raise DryRunRollbackException() + except DryRunRollbackException: + pass sys.exit(0)