diff --git a/layerindex/tools/import_classic.py b/layerindex/tools/import_classic.py index 61f5f90..c006e24 100755 --- a/layerindex/tools/import_classic.py +++ b/layerindex/tools/import_classic.py @@ -27,6 +27,10 @@ import recipeparse logger = utils.logger_create('LayerIndexUpdate') +class DryRunRollbackException(Exception): + pass + + def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir): fn = str(os.path.join(path, recipe.filename)) try: @@ -154,55 +158,51 @@ def main(): # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') - transaction.enter_transaction_management() - transaction.managed(True) try: - layerdir_start = os.path.normpath(oeclassicpath) + os.sep - layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) - layermachines = Machine.objects.filter(layerbranch=layerbranch) - layerdistros = Distro.objects.filter(layerbranch=layerbranch) - layerappends = BBAppend.objects.filter(layerbranch=layerbranch) - layerclasses = BBClass.objects.filter(layerbranch=layerbranch) + with transaction.atomic(): + layerdir_start = os.path.normpath(oeclassicpath) + os.sep + layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) + layermachines = Machine.objects.filter(layerbranch=layerbranch) + layerdistros = Distro.objects.filter(layerbranch=layerbranch) + layerappends = BBAppend.objects.filter(layerbranch=layerbranch) + layerclasses = BBClass.objects.filter(layerbranch=layerbranch) - try: - config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch) - except recipeparse.RecipeParseError as e: - logger.error(str(e)) - transaction.rollback() - sys.exit(1) + try: + config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch) + except recipeparse.RecipeParseError as e: + logger.error(str(e)) + sys.exit(1) - layerrecipes.delete() - layermachines.delete() - layerdistros.delete() - layerappends.delete() - layerclasses.delete() - for root, dirs, files in os.walk(oeclassicpath): - if '.git' in dirs: - dirs.remove('.git') - for f in files: - fullpath = os.path.join(root, f) - (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) - if typename == 'recipe': - recipe = ClassicRecipe() - recipe.layerbranch = layerbranch - recipe.filename = filename - recipe.filepath = filepath - update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath) - recipe.save() + layerrecipes.delete() + layermachines.delete() + layerdistros.delete() + layerappends.delete() + layerclasses.delete() + for root, dirs, files in os.walk(oeclassicpath): + if '.git' in dirs: + dirs.remove('.git') + for f in files: + fullpath = os.path.join(root, f) + (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) + if typename == 'recipe': + recipe = ClassicRecipe() + recipe.layerbranch = layerbranch + recipe.filename = filename + recipe.filepath = filepath + update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath) + recipe.save() - layerbranch.vcs_last_fetch = datetime.now() - layerbranch.save() + layerbranch.vcs_last_fetch = datetime.now() + layerbranch.save() - if options.dryrun: - transaction.rollback() - else: - transaction.commit() + if options.dryrun: + raise DryRunRollbackException() + except DryRunRollbackException: + pass except: import traceback traceback.print_exc() - transaction.rollback() finally: - transaction.leave_transaction_management() tinfoil.shutdown() shutil.rmtree(tempdir) diff --git a/layerindex/tools/import_classic_wiki.py b/layerindex/tools/import_classic_wiki.py index 48c71e9..fb516f1 100755 --- a/layerindex/tools/import_classic_wiki.py +++ b/layerindex/tools/import_classic_wiki.py @@ -21,6 +21,10 @@ import logging logger = utils.logger_create('LayerIndexImport') +class DryRunRollbackException(Exception): + pass + + def read_page(site, path): ret = {} import httplib @@ -98,103 +102,97 @@ def main(): recipes_ai = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_A-I?action=raw") recipes_jz = read_page("www.openembedded.org", "/wiki/OE-Classic_Recipes_J-Z?action=raw") - transaction.enter_transaction_management() - transaction.managed(True) try: - recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) - for pn, comment in recipes.items(): - newpn = '' - newlayer = '' - status = 'U' - comment = comment.strip(' -') - if 'provided by' in comment: - res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - if pn.endswith('-native') or pn.endswith('-cross'): - status = 'P' - else: + with transaction.atomic(): + recipes = dict(list(recipes_ai.items()) + list(recipes_jz.items())) + for pn, comment in recipes.items(): + newpn = '' + newlayer = '' + status = 'U' + comment = comment.strip(' -') + if 'provided by' in comment: + res = re.match(r'[a-zA-Z- ]*provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) + if pn.endswith('-native') or pn.endswith('-cross'): + status = 'P' + else: + status = 'R' + elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: + res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if not res: + res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if not res: + res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) status = 'R' - elif 'replaced by' in comment or 'renamed to' in comment or ' is in ' in comment: - res = re.match(r'.*replaced by ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'.*renamed to ([a-zA-Z0-9-.]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if not res: - res = re.match(r'([a-zA-Z0-9-.]*) is in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'R' - elif 'obsolete' in comment or 'superseded' in comment: - res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - elif comment.startswith('superseded by'): - comment = comment[14:] - elif comment.startswith('obsolete'): - comment = comment[9:] - status = 'O' - elif 'PACKAGECONFIG' in comment: - res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) - if res: - newpn = res.group(1) - newlayer = res.group(2) - comment = res.group(3) - status = 'C' + elif 'obsolete' in comment or 'superseded' in comment: + res = re.match(r'provided by ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) + elif comment.startswith('superseded by'): + comment = comment[14:] + elif comment.startswith('obsolete'): + comment = comment[9:] + status = 'O' + elif 'PACKAGECONFIG' in comment: + res = re.match(r'[a-zA-Z ]* PACKAGECONFIG [a-zA-Z ]* to ([a-zA-Z0-9-]*) in ([a-zA-Z0-9-]*)(.*)', comment) + if res: + newpn = res.group(1) + newlayer = res.group(2) + comment = res.group(3) + status = 'C' - if newlayer: - if newlayer.lower() == 'oe-core': - newlayer = 'openembedded-core' + if newlayer: + if newlayer.lower() == 'oe-core': + newlayer = 'openembedded-core' - # Remove all links from comments because they'll be picked up as categories - comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) - # Split out categories - categories = re.findall(r'\[([^]]*)\]', comment) - for cat in categories: - comment = comment.replace('[%s]' % cat, '') - if '(GPE)' in comment or pn.startswith('gpe'): - categories.append('GPE') - comment = comment.replace('(GPE)', '') + # Remove all links from comments because they'll be picked up as categories + comment = re.sub(r'\[(http[^[]*)\]', r'\1', comment) + # Split out categories + categories = re.findall(r'\[([^]]*)\]', comment) + for cat in categories: + comment = comment.replace('[%s]' % cat, '') + if '(GPE)' in comment or pn.startswith('gpe'): + categories.append('GPE') + comment = comment.replace('(GPE)', '') - comment = comment.strip('- ') + comment = comment.strip('- ') - logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) + logger.debug("%s|%s|%s|%s|%s|%s" % (pn, status, newpn, newlayer, categories, comment)) - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn) - if recipequery: - for recipe in recipequery: - recipe.cover_layerbranch = None - if newlayer: - res = list(LayerItem.objects.filter(name=newlayer)[:1]) - if res: - newlayeritem = res[0] - recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') - else: - logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) - recipe.cover_pn = newpn - recipe.cover_status = status - recipe.cover_verified = True - recipe.cover_comment = comment - recipe.classic_category = " ".join(categories) - recipe.save() - else: - logger.info('No OE-Classic recipe with name "%s" count be found' % pn) - sys.exit(1) + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(pn=pn) + if recipequery: + for recipe in recipequery: + recipe.cover_layerbranch = None + if newlayer: + res = list(LayerItem.objects.filter(name=newlayer)[:1]) + if res: + newlayeritem = res[0] + recipe.cover_layerbranch = newlayeritem.get_layerbranch('master') + else: + logger.info('Replacement layer "%s" for %s could not be found' % (newlayer, pn)) + recipe.cover_pn = newpn + recipe.cover_status = status + recipe.cover_verified = True + recipe.cover_comment = comment + recipe.classic_category = " ".join(categories) + recipe.save() + else: + logger.info('No OE-Classic recipe with name "%s" count be found' % pn) + sys.exit(1) - if options.dryrun: - transaction.rollback() - else: - transaction.commit() - except: - transaction.rollback() - raise - finally: - transaction.leave_transaction_management() + if options.dryrun: + raise DryRunRollbackException() + except DryRunRollbackException: + pass sys.exit(0) diff --git a/layerindex/tools/import_wiki_layers.py b/layerindex/tools/import_wiki_layers.py index f9f9457..bcf57f3 100755 --- a/layerindex/tools/import_wiki_layers.py +++ b/layerindex/tools/import_wiki_layers.py @@ -20,6 +20,10 @@ import utils logger = utils.logger_create('LayerIndexImport') +class DryRunRollbackException(Exception): + pass + + def main(): parser = optparse.OptionParser( @@ -45,9 +49,7 @@ def main(): readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') master_branch = utils.get_branch('master') core_layer = None - transaction.enter_transaction_management() - transaction.managed(True) - try: + with transaction.atomic(): for line in data.splitlines(): if line.startswith('{|'): in_table = True @@ -140,12 +142,6 @@ def main(): layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() - transaction.commit() - except: - transaction.rollback() - raise - finally: - transaction.leave_transaction_management() else: logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) diff --git a/layerindex/tools/update_classic_status.py b/layerindex/tools/update_classic_status.py index ade2461..f980e64 100755 --- a/layerindex/tools/update_classic_status.py +++ b/layerindex/tools/update_classic_status.py @@ -19,6 +19,9 @@ import logging logger = utils.logger_create('LayerIndexClassicUpdate') +class DryRunRollbackException(Exception): + pass + def main(): @@ -62,61 +65,55 @@ def main(): logger.error("Specified branch %s does not exist in database" % options.branch) sys.exit(1) - transaction.enter_transaction_management() - transaction.managed(True) try: - def recipe_pn_query(pn): - return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference') + with transaction.atomic(): + def recipe_pn_query(pn): + return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('layerbranch__layer__index_preference') - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N']) - for recipe in recipequery: - replquery = recipe_pn_query(recipe.pn) - found = False - for replrecipe in replquery: - logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_status = 'D' - recipe.cover_verified = False - recipe.save() - found = True - break - if not found: - if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): - searchpn, _, suffix = recipe.pn.rpartition('-') - replquery = recipe_pn_query(searchpn) - for replrecipe in replquery: - if suffix in replrecipe.bbclassextend.split(): - logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'P' - recipe.cover_verified = False - recipe.save() - found = True - break - if not found and recipe.pn.endswith('-nativesdk'): - searchpn, _, _ = recipe.pn.rpartition('-') - replquery = recipe_pn_query('nativesdk-%s' % searchpn) + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(cover_status__in=['U', 'N']) + for recipe in recipequery: + replquery = recipe_pn_query(recipe.pn) + found = False + for replrecipe in replquery: + logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_status = 'D' + recipe.cover_verified = False + recipe.save() + found = True + break + if not found: + if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): + searchpn, _, suffix = recipe.pn.rpartition('-') + replquery = recipe_pn_query(searchpn) for replrecipe in replquery: - logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'R' - recipe.cover_verified = False - recipe.save() - found = True - break + if suffix in replrecipe.bbclassextend.split(): + logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'P' + recipe.cover_verified = False + recipe.save() + found = True + break + if not found and recipe.pn.endswith('-nativesdk'): + searchpn, _, _ = recipe.pn.rpartition('-') + replquery = recipe_pn_query('nativesdk-%s' % searchpn) + for replrecipe in replquery: + logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'R' + recipe.cover_verified = False + recipe.save() + found = True + break - if options.dryrun: - transaction.rollback() - else: - transaction.commit() - except: - transaction.rollback() - raise - finally: - transaction.leave_transaction_management() + if options.dryrun: + raise DryRunRollbackException() + except DryRunRollbackException: + pass sys.exit(0)