diff --git a/layerindex/tools/update_classic_status.py b/layerindex/tools/update_classic_status.py index a191e34..1262c45 100755 --- a/layerindex/tools/update_classic_status.py +++ b/layerindex/tools/update_classic_status.py @@ -16,6 +16,7 @@ import argparse import re import utils import logging +import json logger = utils.logger_create('LayerIndexComparisonUpdate') @@ -23,6 +24,22 @@ class DryRunRollbackException(Exception): pass +def export(args, layerbranch, skiplist): + from layerindex.models import ClassicRecipe + from django.db.models import F + jscoverlist = [] + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch, deleted=False).order_by('pn').values( + 'pn', 'cover_pn', 'cover_status', 'cover_comment', 'classic_category', cover_layer=F('cover_layerbranch__layer__name')) + for recipe in recipequery: + if recipe['pn'] in skiplist: + logger.debug('Skipping %s' % recipe.pn) + continue + jscoverlist.append(recipe) + jsdata = {'coverlist': jscoverlist} + with open(args.export_data, 'w') as f: + json.dump(jsdata, f, indent=4) + + def main(): parser = argparse.ArgumentParser(description='Comparison recipe cover status update tool') @@ -46,6 +63,15 @@ def main(): parser.add_argument('-q', '--quiet', action='store_const', const=logging.ERROR, dest='loglevel', help='Hide all output except error messages') + parser.add_argument('-i', '--import-data', + metavar='FILE', + help='Import cover status data') + parser.add_argument('--ignore-missing', + action='store_true', + help='Do not warn if a recipe is missing when importing cover status data') + parser.add_argument('--export-data', + metavar='FILE', + help='Export cover status data') args = parser.parse_args() @@ -55,6 +81,10 @@ def main(): logger.setLevel(args.loglevel) + if args.import_data and args.export_data: + logger.error('--i/--import-data and --export-data are mutually exclusive') + sys.exit(1) + layer = LayerItem.objects.filter(name=args.layer).first() if not layer: logger.error('Specified layer %s does not exist in database' % args.layer) @@ -65,6 +95,15 @@ def main(): logger.error("Specified branch %s does not exist in database" % args.branch) sys.exit(1) + if args.skip: + skiplist = args.skip.split(',') + else: + skiplist = [] + + if args.export_data: + export(args, layerbranch, skiplist) + sys.exit(0) + updateobj = None if args.update: updateobj = Update.objects.filter(id=int(args.update)).first() @@ -72,72 +111,133 @@ def main(): logger.error("Specified update id %s does not exist in database" % args.update) sys.exit(1) - if args.skip: - skiplist = args.skip.split(',') - else: - skiplist = [] try: with transaction.atomic(): def recipe_pn_query(pn): return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('-layerbranch__layer__index_preference') - recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(deleted=False).filter(cover_status__in=['U', 'N']) - for recipe in recipequery: - if recipe.pn in skiplist: - logger.debug('Skipping %s' % recipe.pn) - continue - updated = False - sanepn = recipe.pn.lower().replace('_', '-') - replquery = recipe_pn_query(sanepn) - found = False - for replrecipe in replquery: - logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'D' - recipe.cover_verified = False - recipe.save() - updated = True - found = True - break - if not found: - if layerbranch.layer.name == 'oe-classic': - if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): - searchpn, _, suffix = recipe.pn.rpartition('-') - replquery = recipe_pn_query(searchpn) - for replrecipe in replquery: - if suffix in replrecipe.bbclassextend.split(): - logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'P' - recipe.cover_verified = False - recipe.save() - updated = True - found = True - break - if not found and recipe.pn.endswith('-nativesdk'): - searchpn, _, _ = recipe.pn.rpartition('-') - replquery = recipe_pn_query('nativesdk-%s' % searchpn) + if args.import_data: + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch) + layerbranches = {} + with open(args.import_data, 'r') as f: + jsdata = json.load(f) + for jsitem in jsdata['coverlist']: + changed = False + pn = jsitem.pop('pn') + recipe = recipequery.filter(pn=pn).first() + if not recipe: + if not args.ignore_missing: + logger.warning('Could not find recipe %s in %s' % (pn, layerbranch)) + continue + cover_layer = jsitem.pop('cover_layer', None) + if cover_layer: + orig_layerbranch = recipe.cover_layerbranch + recipe.cover_layerbranch = layerbranches.get(cover_layer, None) + if recipe.cover_layerbranch is None: + recipe.cover_layerbranch = LayerBranch.objects.filter(branch__name='master', layer__name=cover_layer).first() + if recipe.cover_layerbranch is None: + logger.warning('Could not find cover layer %s in master branch' % cover_layer) + else: + layerbranches[cover_layer] = recipe.cover_layerbranch + if orig_layerbranch != recipe.cover_layerbranch: + changed = True + elif recipe.cover_layerbranch is not None: + recipe.cover_layerbranch = None + changed = True + valid_fields = [fld.name for fld in ClassicRecipe._meta.get_fields()] + for fieldname, value in jsitem.items(): + if fieldname in valid_fields: + if getattr(recipe, fieldname) != value: + setattr(recipe, fieldname, value) + changed = True + else: + logger.error('Invalid field %s' % fieldname) + sys.exit(1) + if changed: + logger.info('Updating %s' % pn) + utils.validate_fields(recipe) + recipe.save() + else: + recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(deleted=False).filter(cover_status__in=['U', 'N']) + for recipe in recipequery: + if recipe.pn in skiplist: + logger.debug('Skipping %s' % recipe.pn) + continue + updated = False + sanepn = recipe.pn.lower().replace('_', '-') + replquery = recipe_pn_query(sanepn) + found = False + for replrecipe in replquery: + logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'D' + recipe.cover_verified = False + recipe.save() + updated = True + found = True + break + if not found: + if layerbranch.layer.name == 'oe-classic': + if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): + searchpn, _, suffix = recipe.pn.rpartition('-') + replquery = recipe_pn_query(searchpn) for replrecipe in replquery: - logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'R' - recipe.cover_verified = False - recipe.save() - updated = True - found = True - break - else: - if recipe.source_set.exists(): - source0 = recipe.source_set.first() - if 'pypi.' in source0.url or 'pythonhosted.org' in source0.url: - attempts = ['python3-%s' % sanepn, 'python-%s' % sanepn] - if sanepn.startswith('py'): - attempts.extend(['python3-%s' % sanepn[2:], 'python-%s' % sanepn[2:]]) - for attempt in attempts: - replquery = recipe_pn_query(attempt) + if suffix in replrecipe.bbclassextend.split(): + logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'P' + recipe.cover_verified = False + recipe.save() + updated = True + found = True + break + if not found and recipe.pn.endswith('-nativesdk'): + searchpn, _, _ = recipe.pn.rpartition('-') + replquery = recipe_pn_query('nativesdk-%s' % searchpn) + for replrecipe in replquery: + logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'R' + recipe.cover_verified = False + recipe.save() + updated = True + found = True + break + else: + if recipe.source_set.exists(): + source0 = recipe.source_set.first() + if 'pypi.' in source0.url or 'pythonhosted.org' in source0.url: + attempts = ['python3-%s' % sanepn, 'python-%s' % sanepn] + if sanepn.startswith('py'): + attempts.extend(['python3-%s' % sanepn[2:], 'python-%s' % sanepn[2:]]) + for attempt in attempts: + replquery = recipe_pn_query(attempt) + for replrecipe in replquery: + logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'D' + recipe.cover_verified = False + recipe.save() + updated = True + found = True + break + if found: + break + if not found: + recipe.classic_category = 'python' + recipe.save() + updated = True + elif 'cpan.org' in source0.url: + perlpn = sanepn + if perlpn.startswith('perl-'): + perlpn = perlpn[5:] + if not (perlpn.startswith('lib') and perlpn.endswith('-perl')): + perlpn = 'lib%s-perl' % perlpn + replquery = recipe_pn_query(perlpn) for replrecipe in replquery: logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) recipe.cover_layerbranch = replrecipe.layerbranch @@ -148,89 +248,66 @@ def main(): updated = True found = True break - if found: + if not found: + recipe.classic_category = 'perl' + recipe.save() + updated = True + elif 'kde.org' in source0.url or 'github.com/KDE' in source0.url: + recipe.classic_category = 'kde' + recipe.save() + updated = True + if not found: + if recipe.pn.startswith('R-'): + recipe.classic_category = 'R' + recipe.save() + updated = True + elif recipe.pn.startswith('rubygem-'): + recipe.classic_category = 'ruby' + recipe.save() + updated = True + elif recipe.pn.startswith('jdk-'): + sanepn = sanepn[4:] + replquery = recipe_pn_query(sanepn) + for replrecipe in replquery: + logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'D' + recipe.cover_verified = False + recipe.save() + updated = True + found = True break - if not found: - recipe.classic_category = 'python' + recipe.classic_category = 'java' recipe.save() updated = True - elif 'cpan.org' in source0.url: - perlpn = sanepn - if perlpn.startswith('perl-'): - perlpn = perlpn[5:] - if not (perlpn.startswith('lib') and perlpn.endswith('-perl')): - perlpn = 'lib%s-perl' % perlpn - replquery = recipe_pn_query(perlpn) - for replrecipe in replquery: - logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'D' - recipe.cover_verified = False + elif recipe.pn.startswith('golang-'): + if recipe.pn.startswith('golang-github-'): + sanepn = 'go-' + sanepn[14:] + else: + sanepn = 'go-' + sanepn[7:] + replquery = recipe_pn_query(sanepn) + for replrecipe in replquery: + logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) + recipe.cover_layerbranch = replrecipe.layerbranch + recipe.cover_pn = replrecipe.pn + recipe.cover_status = 'D' + recipe.cover_verified = False + recipe.save() + updated = True + found = True + break + recipe.classic_category = 'go' recipe.save() updated = True - found = True - break - if not found: + elif recipe.pn.startswith('gnome-'): + recipe.classic_category = 'gnome' + recipe.save() + updated = True + elif recipe.pn.startswith('perl-'): recipe.classic_category = 'perl' recipe.save() updated = True - elif 'kde.org' in source0.url or 'github.com/KDE' in source0.url: - recipe.classic_category = 'kde' - recipe.save() - updated = True - if not found: - if recipe.pn.startswith('R-'): - recipe.classic_category = 'R' - recipe.save() - updated = True - elif recipe.pn.startswith('rubygem-'): - recipe.classic_category = 'ruby' - recipe.save() - updated = True - elif recipe.pn.startswith('jdk-'): - sanepn = sanepn[4:] - replquery = recipe_pn_query(sanepn) - for replrecipe in replquery: - logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'D' - recipe.cover_verified = False - recipe.save() - updated = True - found = True - break - recipe.classic_category = 'java' - recipe.save() - updated = True - elif recipe.pn.startswith('golang-'): - if recipe.pn.startswith('golang-github-'): - sanepn = 'go-' + sanepn[14:] - else: - sanepn = 'go-' + sanepn[7:] - replquery = recipe_pn_query(sanepn) - for replrecipe in replquery: - logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) - recipe.cover_layerbranch = replrecipe.layerbranch - recipe.cover_pn = replrecipe.pn - recipe.cover_status = 'D' - recipe.cover_verified = False - recipe.save() - updated = True - found = True - break - recipe.classic_category = 'go' - recipe.save() - updated = True - elif recipe.pn.startswith('gnome-'): - recipe.classic_category = 'gnome' - recipe.save() - updated = True - elif recipe.pn.startswith('perl-'): - recipe.classic_category = 'perl' - recipe.save() - updated = True if updated and updateobj: rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe) rupdate.link_updated = True