mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 12:49:01 +02:00
87e27a26d5
Add a script to import "recipe" information from other distro metadata, based on the import_classic.py script. At the moment, this assumes a directory where each subdirectory is a package directory containing a spec file; this would be suitable for distributions such as Fedora assuming you have all of the package repos checked out locally. Since you can add additional information to these records (the cover fields pointing to matching recipes), existing records are updated rather than deleting everything and re-importing, and we only mark records as deleted rather than actually deleting them (in case you accidentally point the script at an empty directory or similar). Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
485 lines
20 KiB
Python
Executable File
485 lines
20 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Import other distro information
|
|
#
|
|
# Copyright (C) 2013, 2018 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
|
|
import sys
|
|
import os
|
|
import argparse
|
|
import logging
|
|
from datetime import datetime
|
|
import re
|
|
import tempfile
|
|
import glob
|
|
import shutil
|
|
import subprocess
|
|
from distutils.version import LooseVersion
|
|
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), 'lib')))
|
|
|
|
import utils
|
|
import recipeparse
|
|
|
|
logger = utils.logger_create('LayerIndexOtherDistro')
|
|
|
|
|
|
class DryRunRollbackException(Exception):
|
|
pass
|
|
|
|
|
|
def get_gourl(goipath):
|
|
# Much more crude than the original implementation
|
|
gourl = goipath
|
|
if not '://' in gourl:
|
|
gourl = 'https://' + gourl
|
|
return gourl
|
|
|
|
|
|
def update_recipe_file(path, recipe, repodir, raiseexceptions=False):
|
|
from layerindex.models import Patch, Source
|
|
from django.db import DatabaseError
|
|
|
|
# yes, yes, I know this is all crude as hell, but it gets the job done.
|
|
# At the end of the day we are scraping the spec file, we aren't trying to build it
|
|
|
|
try:
|
|
logger.debug('Updating recipe %s' % path)
|
|
recipe.pn = os.path.splitext(recipe.filename)[0]
|
|
with open(path, 'r', errors='surrogateescape') as f:
|
|
indesc = False
|
|
desc = []
|
|
patches = []
|
|
sources = []
|
|
values = {}
|
|
defines = {'__awk': 'awk',
|
|
'__python3': 'python3',
|
|
'__python2': 'python',
|
|
'__python': 'python',
|
|
'__sed': 'sed',
|
|
'__perl': 'perl',
|
|
'__id_u': 'id -u',
|
|
'__cat': 'cat',
|
|
'__grep': 'grep',
|
|
'_bindir': '/usr/bin',
|
|
'_sbindir': '/usr/sbin',
|
|
'_datadir': '/usr/share',
|
|
'_docdir': '%{datadir}/doc',
|
|
'_defaultdocdir': '%{datadir}/doc',
|
|
'_pkgdocdir': '%{_docdir}/%{name}'
|
|
}
|
|
globaldefs = {}
|
|
|
|
def expand(expr):
|
|
inmacro = 0
|
|
inshell = 0
|
|
lastch = None
|
|
macroexpr = ''
|
|
shellexpr = ''
|
|
outstr = ''
|
|
for i, ch in enumerate(expr):
|
|
if inshell:
|
|
if ch == '(':
|
|
inshell += 1
|
|
elif ch == ')':
|
|
inshell -= 1
|
|
if inshell == 0:
|
|
try:
|
|
shellcmd = expand(shellexpr)
|
|
expanded = subprocess.check_output(shellcmd, shell=True).decode('utf-8').rstrip()
|
|
except Exception as e:
|
|
logger.warning('Failed to execute "%s": %s' % (shellcmd, str(e)))
|
|
expanded = ''
|
|
if expanded:
|
|
outstr += expanded
|
|
lastch = ch
|
|
continue
|
|
shellexpr += ch
|
|
elif inmacro:
|
|
if ch == '}':
|
|
inmacro -= 1
|
|
if inmacro == 0:
|
|
if macroexpr.startswith('?'):
|
|
macrosplit = macroexpr[1:].split(':')
|
|
macrokey = macrosplit[0].lower()
|
|
if macrokey in globaldefs or macrokey in defines or macrokey in values:
|
|
if len(macrosplit) > 1:
|
|
outstr += expand(macrosplit[1])
|
|
else:
|
|
expanded = expand(values.get(macrokey, '') or defines.get(macrokey, '') or globaldefs.get(macrokey, ''))
|
|
if expanded:
|
|
outstr += expanded
|
|
elif macroexpr.startswith('!?'):
|
|
macrosplit = macroexpr[2:].split(':')
|
|
macrokey = macrosplit[0].lower()
|
|
if len(macrosplit) > 1:
|
|
if not (macrokey in globaldefs or macrokey in defines or macrokey in values):
|
|
outstr += expand(macrosplit[1])
|
|
else:
|
|
macrokey = macroexpr.lower()
|
|
expanded = expand(values.get(macrokey, '') or defines.get(macrokey, '') or globaldefs.get(macrokey, ''))
|
|
if expanded:
|
|
outstr += expanded
|
|
else:
|
|
outstr += '%{' + macroexpr + '}'
|
|
lastch = ch
|
|
continue
|
|
macroexpr += ch
|
|
if ch == '{':
|
|
if lastch == '%':
|
|
if inmacro == 0:
|
|
macroexpr = ''
|
|
outstr = outstr[:-1]
|
|
inmacro += 1
|
|
elif ch == '(':
|
|
if lastch == '%':
|
|
if inshell == 0:
|
|
shellexpr = ''
|
|
outstr = outstr[:-1]
|
|
inshell += 1
|
|
if inmacro == 0 and inshell == 0:
|
|
if ch == '%':
|
|
# Handle unbracketed expressions (in which case we eat the rest of the expression)
|
|
if expr[i+1] not in ['{', '%']:
|
|
macrokey = expr[i+1:].split()[0]
|
|
if macrokey in globaldefs or macrokey in defines or macrokey in values:
|
|
expanded = expand(values.get(macrokey, '') or defines.get(macrokey, '') or globaldefs.get(macrokey, ''))
|
|
if expanded:
|
|
outstr += expanded
|
|
break
|
|
if ch == '%' and lastch == '%':
|
|
# %% is a literal %, so skip this one (and don't allow this to happen again if the next char is a %)
|
|
lastch = ''
|
|
continue
|
|
outstr += ch
|
|
lastch = ch
|
|
return outstr
|
|
|
|
def eval_cond(cond, condtype):
|
|
negate = False
|
|
if condtype == '%if':
|
|
if cond.startswith('(') and cond.endswith(')'):
|
|
cond = cond[1:-1].strip()
|
|
if cond.startswith('!'):
|
|
cond = cond[1:].lstrip()
|
|
negate = True
|
|
res = False
|
|
try:
|
|
if int(cond):
|
|
res = True
|
|
except ValueError:
|
|
pass
|
|
elif condtype in ['%ifos', '%ifnos']:
|
|
if condtype == '%ifnos':
|
|
negate = True
|
|
# Assume linux
|
|
res = ('linux' in cond.split())
|
|
elif condtype in ['%ifarch', '%ifnarch']:
|
|
if condtype == '%ifnarch':
|
|
negate = True
|
|
res = ('x86_64' in cond.split())
|
|
else:
|
|
raise Exception('Unhandled conditional type "%s"' % condtype)
|
|
if negate:
|
|
return not res
|
|
else:
|
|
return res
|
|
|
|
conds = []
|
|
reading = True
|
|
for line in f:
|
|
if line.startswith('%package'):
|
|
# Assume it's OK to stop when we hit the first package
|
|
break
|
|
if line.startswith(('%gometa', '%gocraftmeta')):
|
|
goipath = globaldefs.get('goipath', '')
|
|
if not goipath:
|
|
goipath = globaldefs.get('gobaseipath', '')
|
|
if goipath:
|
|
# We could use a python translation of the full logic from
|
|
# the RPM macros to get this - but it turns out the spec files
|
|
# (in Fedora at least) already use these processed names, so
|
|
# there's no point
|
|
globaldefs['goname'] = os.path.splitext(os.path.basename(path))[0]
|
|
globaldefs['gourl'] = get_gourl(goipath)
|
|
elif line.startswith('%if') and ' ' in line:
|
|
conds.append(reading)
|
|
splitline = line.split()
|
|
cond = expand(' '.join(splitline[1:]))
|
|
if not eval_cond(cond, splitline[0]):
|
|
reading = False
|
|
elif line.startswith('%else'):
|
|
reading = not reading
|
|
elif line.startswith('%endif'):
|
|
reading = conds.pop()
|
|
if not reading:
|
|
continue
|
|
if line.startswith(('%define', '%global')):
|
|
linesplit = line.split()
|
|
name = linesplit[1].lower()
|
|
value = ' '.join(linesplit[2:])
|
|
if value.lower() == '%{' + name + '}':
|
|
# STOP THE INSANITY!
|
|
# (as seen in cups/cups.spec in Fedora)
|
|
continue
|
|
if line.startswith('%global'):
|
|
globaldefs[name] = expand(value)
|
|
else:
|
|
defines[name] = value
|
|
continue
|
|
elif line.startswith('%undefine'):
|
|
linesplit = line.split()
|
|
name = linesplit[1].lower()
|
|
if name in globaldefs:
|
|
del globaldefs[name]
|
|
if name in defines:
|
|
del defines[name]
|
|
continue
|
|
elif line.strip() == '%description':
|
|
indesc = True
|
|
continue
|
|
if indesc:
|
|
# We want to stop parsing the description when we hit another macro,
|
|
# but we do want to allow bracketed macro expressions within the description
|
|
# (e.g. %{name})
|
|
if line.startswith('%') and len(line) > 1 and line[1] != '{':
|
|
indesc = False
|
|
elif not line.startswith('#'):
|
|
desc.append(line)
|
|
|
|
if ':' in line and not line.startswith('%'):
|
|
key, value = line.split(':', 1)
|
|
key = key.rstrip().lower()
|
|
value = value.strip()
|
|
values[key] = expand(value)
|
|
|
|
for key, value in values.items():
|
|
if key == 'name':
|
|
recipe.pn = expand(value)
|
|
elif key == 'version':
|
|
recipe.pv = expand(value)
|
|
elif key == 'summary':
|
|
recipe.summary = expand(value.strip('"\''))
|
|
elif key == 'group':
|
|
recipe.section = expand(value)
|
|
elif key == 'url':
|
|
recipe.homepage = expand(value)
|
|
elif key == 'license':
|
|
recipe.license = expand(value)
|
|
elif key.startswith('patch'):
|
|
patches.append(expand(value))
|
|
elif key.startswith('source'):
|
|
sources.append(expand(value))
|
|
|
|
recipe.description = expand(' '.join(desc).rstrip())
|
|
recipe.save()
|
|
|
|
saved_patches = []
|
|
for patchfn in patches:
|
|
patchpath = os.path.join(os.path.relpath(os.path.dirname(path), repodir), patchfn)
|
|
patch, _ = Patch.objects.get_or_create(recipe=recipe, path=patchpath)
|
|
patch.src_path = patchfn
|
|
patch.save()
|
|
saved_patches.append(patch.id)
|
|
recipe.patch_set.exclude(id__in=saved_patches).delete()
|
|
|
|
existing_ids = list(recipe.source_set.values_list('id', flat=True))
|
|
for src in sources:
|
|
srcobj, _ = Source.objects.get_or_create(recipe=recipe, url=src)
|
|
srcobj.save()
|
|
if srcobj.id in existing_ids:
|
|
existing_ids.remove(srcobj.id)
|
|
# Need to delete like this because some spec files have a lot of sources!
|
|
for idv in existing_ids:
|
|
Source.objects.filter(id=idv).delete()
|
|
except DatabaseError:
|
|
raise
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except BaseException as e:
|
|
if raiseexceptions:
|
|
raise
|
|
else:
|
|
if not recipe.pn:
|
|
recipe.pn = recipe.filename[:-3].split('_')[0]
|
|
logger.error("Unable to read %s: %s", path, str(e))
|
|
|
|
|
|
def check_branch_layer(args):
|
|
from layerindex.models import LayerItem, LayerBranch
|
|
|
|
branch = utils.get_branch(args.branch)
|
|
if not branch:
|
|
logger.error("Specified branch %s is not valid" % args.branch)
|
|
return 1, None
|
|
|
|
res = list(LayerItem.objects.filter(name=args.layer)[:1])
|
|
if res:
|
|
layer = res[0]
|
|
else:
|
|
logger.error('Cannot find specified layer "%s"' % args.layer)
|
|
return 1, None
|
|
|
|
layerbranch = layer.get_layerbranch(args.branch)
|
|
if not layerbranch:
|
|
# LayerBranch doesn't exist for this branch, create it
|
|
layerbranch = LayerBranch()
|
|
layerbranch.layer = layer
|
|
layerbranch.branch = branch
|
|
layerbranch.save()
|
|
|
|
return 0, layerbranch
|
|
|
|
|
|
def import_pkgspec(args):
|
|
utils.setup_django()
|
|
import settings
|
|
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
|
|
from django.db import transaction
|
|
|
|
ret, layerbranch = check_branch_layer(args)
|
|
if ret:
|
|
return ret
|
|
|
|
metapath = args.pkgdir
|
|
|
|
try:
|
|
with transaction.atomic():
|
|
layerrecipes = ClassicRecipe.objects.filter(layerbranch=layerbranch)
|
|
|
|
existing = list(layerrecipes.filter(deleted=False).values_list('filepath', 'filename'))
|
|
for entry in os.listdir(metapath):
|
|
if os.path.exists(os.path.join(metapath, entry, 'dead.package')):
|
|
logger.info('Skipping dead package %s' % entry)
|
|
continue
|
|
specfiles = glob.glob(os.path.join(metapath, entry, '*.spec'))
|
|
if specfiles:
|
|
for specfile in specfiles:
|
|
specfn = os.path.basename(specfile)
|
|
specpath = os.path.relpath(os.path.dirname(specfile), metapath)
|
|
recipe, created = ClassicRecipe.objects.get_or_create(layerbranch=layerbranch, filepath=specpath, filename=specfn)
|
|
if created:
|
|
logger.info('Importing %s' % specfn)
|
|
elif recipe.deleted:
|
|
logger.info('Restoring and updating %s' % specpath)
|
|
recipe.deleted = False
|
|
else:
|
|
logger.info('Updating %s' % specpath)
|
|
recipe.layerbranch = layerbranch
|
|
recipe.filename = specfn
|
|
recipe.filepath = specpath
|
|
update_recipe_file(specfile, recipe, metapath)
|
|
recipe.save()
|
|
existingentry = (specpath, specfn)
|
|
if existingentry in existing:
|
|
existing.remove(existingentry)
|
|
else:
|
|
logger.warn('Missing spec file in %s' % os.path.join(metapath, entry))
|
|
|
|
if existing:
|
|
fpaths = ['%s/%s' % (pth, fn) for pth, fn in existing]
|
|
logger.info('Marking as deleted: %s' % ', '.join(fpaths))
|
|
for entry in existing:
|
|
layerrecipes.filter(filepath=entry[0], filename=entry[1]).update(deleted=True)
|
|
|
|
layerbranch.vcs_last_fetch = datetime.now()
|
|
layerbranch.save()
|
|
|
|
if args.dry_run:
|
|
raise DryRunRollbackException()
|
|
except DryRunRollbackException:
|
|
pass
|
|
except:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return 1
|
|
|
|
return 0
|
|
|
|
|
|
def try_specfile(args):
|
|
utils.setup_django()
|
|
import settings
|
|
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
|
|
from django.db import transaction
|
|
|
|
ret, layerbranch = check_branch_layer(args)
|
|
if ret:
|
|
return ret
|
|
|
|
specfile = args.specfile
|
|
metapath = os.path.dirname(specfile)
|
|
|
|
try:
|
|
with transaction.atomic():
|
|
recipe = ClassicRecipe()
|
|
recipe.layerbranch = layerbranch
|
|
recipe.filename = os.path.basename(specfile)
|
|
recipe.filepath = os.path.relpath(os.path.dirname(specfile), metapath)
|
|
update_recipe_file(specfile, recipe, metapath, raiseexceptions=True)
|
|
recipe.save()
|
|
for f in Recipe._meta.get_fields():
|
|
if not (f.auto_created and f.is_relation):
|
|
print('%s: %s' % (f.name, getattr(recipe, f.name)))
|
|
if recipe.source_set.exists():
|
|
print('sources:')
|
|
for src in recipe.source_set.all():
|
|
print(' * %s' % src.url)
|
|
|
|
raise DryRunRollbackException()
|
|
except DryRunRollbackException:
|
|
pass
|
|
except:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return 1
|
|
|
|
return 0
|
|
|
|
|
|
def main():
|
|
|
|
parser = argparse.ArgumentParser(description='OE Layer Index other distro comparison import tool',
|
|
epilog='Use %(prog)s <subcommand> --help to get help on a specific command')
|
|
parser.add_argument('-d', '--debug', help='Enable debug output', action='store_const', const=logging.DEBUG, dest='loglevel', default=logging.INFO)
|
|
parser.add_argument('-q', '--quiet', help='Hide all output except error messages', action='store_const', const=logging.ERROR, dest='loglevel')
|
|
|
|
subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')
|
|
subparsers.required = True
|
|
|
|
|
|
parser_pkgspec = subparsers.add_parser('import-pkgspec',
|
|
help='Import from a local rpm-based distro package directory',
|
|
description='Imports from a local directory containing subdirectories, each of which contains an RPM .spec file for a package')
|
|
parser_pkgspec.add_argument('branch', help='Branch to import into')
|
|
parser_pkgspec.add_argument('layer', help='Layer to import into')
|
|
parser_pkgspec.add_argument('pkgdir', help='Top level directory containing package subdirectories')
|
|
parser_pkgspec.add_argument('-n', '--dry-run', help='Don\'t write any data back to the database', action='store_true')
|
|
parser_pkgspec.set_defaults(func=import_pkgspec)
|
|
|
|
|
|
parser_tryspecfile = subparsers.add_parser('try-specfile',
|
|
help='Test importing from a local RPM spec file',
|
|
description='Tests importing an RPM .spec file for a package')
|
|
parser_tryspecfile.add_argument('branch', help='Branch to import into')
|
|
parser_tryspecfile.add_argument('layer', help='Layer to import into')
|
|
parser_tryspecfile.add_argument('specfile', help='Spec file to try importing')
|
|
parser_tryspecfile.set_defaults(func=try_specfile)
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
logger.setLevel(args.loglevel)
|
|
|
|
ret = args.func(args)
|
|
|
|
return ret
|
|
|
|
if __name__ == "__main__":
|
|
main()
|