MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-19 20:59:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

update_classic_status: add cover status import/export

Browse Source 
Provide a way to export and import other distro recipe coverage in
update_classic_status.py.

(Based on the Clear Linux Dissector commit
ff38e9582093453e13b90e06af125374ca4b0a16).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2019-09-09 10:35:29 +12:00
parent 63446ed8de
commit 085447cecc
1 changed files with 212 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

347
layerindex/tools/update_classic_status.py
View File

@ -16,6 +16,7 @@ import argparse
import re import re
import utils import utils
import logging import logging
import json
logger = utils.logger_create('LayerIndexComparisonUpdate') logger = utils.logger_create('LayerIndexComparisonUpdate')
@ -23,6 +24,22 @@ class DryRunRollbackException(Exception):
pass pass
def export(args, layerbranch, skiplist):
from layerindex.models import ClassicRecipe
from django.db.models import F
jscoverlist = []
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch, deleted=False).order_by('pn').values(
'pn', 'cover_pn', 'cover_status', 'cover_comment', 'classic_category', cover_layer=F('cover_layerbranch__layer__name'))
for recipe in recipequery:
if recipe['pn'] in skiplist:
logger.debug('Skipping %s' % recipe.pn)
continue
jscoverlist.append(recipe)
jsdata = {'coverlist': jscoverlist}
with open(args.export_data, 'w') as f:
json.dump(jsdata, f, indent=4)
def main(): def main():
parser = argparse.ArgumentParser(description='Comparison recipe cover status update tool') parser = argparse.ArgumentParser(description='Comparison recipe cover status update tool')
@ -46,6 +63,15 @@ def main():
parser.add_argument('-q', '--quiet', parser.add_argument('-q', '--quiet',
action='store_const', const=logging.ERROR, dest='loglevel', action='store_const', const=logging.ERROR, dest='loglevel',
help='Hide all output except error messages') help='Hide all output except error messages')
parser.add_argument('-i', '--import-data',
metavar='FILE',
help='Import cover status data')
parser.add_argument('--ignore-missing',
action='store_true',
help='Do not warn if a recipe is missing when importing cover status data')
parser.add_argument('--export-data',
metavar='FILE',
help='Export cover status data')
args = parser.parse_args() args = parser.parse_args()
@ -55,6 +81,10 @@ def main():
logger.setLevel(args.loglevel) logger.setLevel(args.loglevel)
if args.import_data and args.export_data:
logger.error('--i/--import-data and --export-data are mutually exclusive')
sys.exit(1)
layer = LayerItem.objects.filter(name=args.layer).first() layer = LayerItem.objects.filter(name=args.layer).first()
if not layer: if not layer:
logger.error('Specified layer %s does not exist in database' % args.layer) logger.error('Specified layer %s does not exist in database' % args.layer)
@ -65,6 +95,15 @@ def main():
logger.error("Specified branch %s does not exist in database" % args.branch) logger.error("Specified branch %s does not exist in database" % args.branch)
sys.exit(1) sys.exit(1)
if args.skip:
skiplist = args.skip.split(',')
else:
skiplist = []
if args.export_data:
export(args, layerbranch, skiplist)
sys.exit(0)
updateobj = None updateobj = None
if args.update: if args.update:
updateobj = Update.objects.filter(id=int(args.update)).first() updateobj = Update.objects.filter(id=int(args.update)).first()
@ -72,72 +111,133 @@ def main():
logger.error("Specified update id %s does not exist in database" % args.update) logger.error("Specified update id %s does not exist in database" % args.update)
sys.exit(1) sys.exit(1)
if args.skip:
skiplist = args.skip.split(',')
else:
skiplist = []
try: try:
with transaction.atomic(): with transaction.atomic():
def recipe_pn_query(pn): def recipe_pn_query(pn):
return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('-layerbranch__layer__index_preference') return Recipe.objects.filter(layerbranch__branch__name='master').filter(pn=pn).order_by('-layerbranch__layer__index_preference')
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(deleted=False).filter(cover_status__in=['U', 'N']) if args.import_data:
for recipe in recipequery: recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch)
if recipe.pn in skiplist: layerbranches = {}
logger.debug('Skipping %s' % recipe.pn) with open(args.import_data, 'r') as f:
continue jsdata = json.load(f)
updated = False for jsitem in jsdata['coverlist']:
sanepn = recipe.pn.lower().replace('_', '-') changed = False
replquery = recipe_pn_query(sanepn) pn = jsitem.pop('pn')
found = False recipe = recipequery.filter(pn=pn).first()
for replrecipe in replquery: if not recipe:
logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name)) if not args.ignore_missing:
recipe.cover_layerbranch = replrecipe.layerbranch logger.warning('Could not find recipe %s in %s' % (pn, layerbranch))
recipe.cover_pn = replrecipe.pn continue
recipe.cover_status = 'D' cover_layer = jsitem.pop('cover_layer', None)
recipe.cover_verified = False if cover_layer:
recipe.save() orig_layerbranch = recipe.cover_layerbranch
updated = True recipe.cover_layerbranch = layerbranches.get(cover_layer, None)
found = True if recipe.cover_layerbranch is None:
break recipe.cover_layerbranch = LayerBranch.objects.filter(branch__name='master', layer__name=cover_layer).first()
if not found: if recipe.cover_layerbranch is None:
if layerbranch.layer.name == 'oe-classic': logger.warning('Could not find cover layer %s in master branch' % cover_layer)
if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'): else:
searchpn, _, suffix = recipe.pn.rpartition('-') layerbranches[cover_layer] = recipe.cover_layerbranch
replquery = recipe_pn_query(searchpn) if orig_layerbranch != recipe.cover_layerbranch:
for replrecipe in replquery: changed = True
if suffix in replrecipe.bbclassextend.split(): elif recipe.cover_layerbranch is not None:
logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) recipe.cover_layerbranch = None
recipe.cover_layerbranch = replrecipe.layerbranch changed = True
recipe.cover_pn = replrecipe.pn valid_fields = [fld.name for fld in ClassicRecipe._meta.get_fields()]
recipe.cover_status = 'P' for fieldname, value in jsitem.items():
recipe.cover_verified = False if fieldname in valid_fields:
recipe.save() if getattr(recipe, fieldname) != value:
updated = True setattr(recipe, fieldname, value)
found = True changed = True
break else:
if not found and recipe.pn.endswith('-nativesdk'): logger.error('Invalid field %s' % fieldname)
searchpn, _, _ = recipe.pn.rpartition('-') sys.exit(1)
replquery = recipe_pn_query('nativesdk-%s' % searchpn) if changed:
logger.info('Updating %s' % pn)
utils.validate_fields(recipe)
recipe.save()
else:
recipequery = ClassicRecipe.objects.filter(layerbranch=layerbranch).filter(deleted=False).filter(cover_status__in=['U', 'N'])
for recipe in recipequery:
if recipe.pn in skiplist:
logger.debug('Skipping %s' % recipe.pn)
continue
updated = False
sanepn = recipe.pn.lower().replace('_', '-')
replquery = recipe_pn_query(sanepn)
found = False
for replrecipe in replquery:
logger.debug('Matched %s in layer %s' % (recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
if not found:
if layerbranch.layer.name == 'oe-classic':
if recipe.pn.endswith('-native') or recipe.pn.endswith('-nativesdk'):
searchpn, _, suffix = recipe.pn.rpartition('-')
replquery = recipe_pn_query(searchpn)
for replrecipe in replquery: for replrecipe in replquery:
logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) if suffix in replrecipe.bbclassextend.split():
recipe.cover_layerbranch = replrecipe.layerbranch logger.debug('Found BBCLASSEXTEND of %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_pn = replrecipe.pn recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_status = 'R' recipe.cover_pn = replrecipe.pn
recipe.cover_verified = False recipe.cover_status = 'P'
recipe.save() recipe.cover_verified = False
updated = True recipe.save()
found = True updated = True
break found = True
else: break
if recipe.source_set.exists(): if not found and recipe.pn.endswith('-nativesdk'):
source0 = recipe.source_set.first() searchpn, _, _ = recipe.pn.rpartition('-')
if 'pypi.' in source0.url or 'pythonhosted.org' in source0.url: replquery = recipe_pn_query('nativesdk-%s' % searchpn)
attempts = ['python3-%s' % sanepn, 'python-%s' % sanepn] for replrecipe in replquery:
if sanepn.startswith('py'): logger.debug('Found replacement %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
attempts.extend(['python3-%s' % sanepn[2:], 'python-%s' % sanepn[2:]]) recipe.cover_layerbranch = replrecipe.layerbranch
for attempt in attempts: recipe.cover_pn = replrecipe.pn
replquery = recipe_pn_query(attempt) recipe.cover_status = 'R'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
else:
if recipe.source_set.exists():
source0 = recipe.source_set.first()
if 'pypi.' in source0.url or 'pythonhosted.org' in source0.url:
attempts = ['python3-%s' % sanepn, 'python-%s' % sanepn]
if sanepn.startswith('py'):
attempts.extend(['python3-%s' % sanepn[2:], 'python-%s' % sanepn[2:]])
for attempt in attempts:
replquery = recipe_pn_query(attempt)
for replrecipe in replquery:
logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
if found:
break
if not found:
recipe.classic_category = 'python'
recipe.save()
updated = True
elif 'cpan.org' in source0.url:
perlpn = sanepn
if perlpn.startswith('perl-'):
perlpn = perlpn[5:]
if not (perlpn.startswith('lib') and perlpn.endswith('-perl')):
perlpn = 'lib%s-perl' % perlpn
replquery = recipe_pn_query(perlpn)
for replrecipe in replquery: for replrecipe in replquery:
logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch recipe.cover_layerbranch = replrecipe.layerbranch
@ -148,89 +248,66 @@ def main():
updated = True updated = True
found = True found = True
break break
if found: if not found:
recipe.classic_category = 'perl'
recipe.save()
updated = True
elif 'kde.org' in source0.url or 'github.com/KDE' in source0.url:
recipe.classic_category = 'kde'
recipe.save()
updated = True
if not found:
if recipe.pn.startswith('R-'):
recipe.classic_category = 'R'
recipe.save()
updated = True
elif recipe.pn.startswith('rubygem-'):
recipe.classic_category = 'ruby'
recipe.save()
updated = True
elif recipe.pn.startswith('jdk-'):
sanepn = sanepn[4:]
replquery = recipe_pn_query(sanepn)
for replrecipe in replquery:
logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break break
if not found: recipe.classic_category = 'java'
recipe.classic_category = 'python'
recipe.save() recipe.save()
updated = True updated = True
elif 'cpan.org' in source0.url: elif recipe.pn.startswith('golang-'):
perlpn = sanepn if recipe.pn.startswith('golang-github-'):
if perlpn.startswith('perl-'): sanepn = 'go-' + sanepn[14:]
perlpn = perlpn[5:] else:
if not (perlpn.startswith('lib') and perlpn.endswith('-perl')): sanepn = 'go-' + sanepn[7:]
perlpn = 'lib%s-perl' % perlpn replquery = recipe_pn_query(sanepn)
replquery = recipe_pn_query(perlpn) for replrecipe in replquery:
for replrecipe in replquery: logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name)) recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_layerbranch = replrecipe.layerbranch recipe.cover_pn = replrecipe.pn
recipe.cover_pn = replrecipe.pn recipe.cover_status = 'D'
recipe.cover_status = 'D' recipe.cover_verified = False
recipe.cover_verified = False recipe.save()
updated = True
found = True
break
recipe.classic_category = 'go'
recipe.save() recipe.save()
updated = True updated = True
found = True elif recipe.pn.startswith('gnome-'):
break recipe.classic_category = 'gnome'
if not found: recipe.save()
updated = True
elif recipe.pn.startswith('perl-'):
recipe.classic_category = 'perl' recipe.classic_category = 'perl'
recipe.save() recipe.save()
updated = True updated = True
elif 'kde.org' in source0.url or 'github.com/KDE' in source0.url:
recipe.classic_category = 'kde'
recipe.save()
updated = True
if not found:
if recipe.pn.startswith('R-'):
recipe.classic_category = 'R'
recipe.save()
updated = True
elif recipe.pn.startswith('rubygem-'):
recipe.classic_category = 'ruby'
recipe.save()
updated = True
elif recipe.pn.startswith('jdk-'):
sanepn = sanepn[4:]
replquery = recipe_pn_query(sanepn)
for replrecipe in replquery:
logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
recipe.classic_category = 'java'
recipe.save()
updated = True
elif recipe.pn.startswith('golang-'):
if recipe.pn.startswith('golang-github-'):
sanepn = 'go-' + sanepn[14:]
else:
sanepn = 'go-' + sanepn[7:]
replquery = recipe_pn_query(sanepn)
for replrecipe in replquery:
logger.debug('Found match %s to cover %s in layer %s' % (replrecipe.pn, recipe.pn, replrecipe.layerbranch.layer.name))
recipe.cover_layerbranch = replrecipe.layerbranch
recipe.cover_pn = replrecipe.pn
recipe.cover_status = 'D'
recipe.cover_verified = False
recipe.save()
updated = True
found = True
break
recipe.classic_category = 'go'
recipe.save()
updated = True
elif recipe.pn.startswith('gnome-'):
recipe.classic_category = 'gnome'
recipe.save()
updated = True
elif recipe.pn.startswith('perl-'):
recipe.classic_category = 'perl'
recipe.save()
updated = True
if updated and updateobj: if updated and updateobj:
rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe) rupdate, _ = ComparisonRecipeUpdate.objects.get_or_create(update=updateobj, recipe=recipe)
rupdate.link_updated = True rupdate.link_updated = True