diff --git a/layerindex/bulkchange.py b/layerindex/bulkchange.py index 316c711..c06acce 100644 --- a/layerindex/bulkchange.py +++ b/layerindex/bulkchange.py @@ -2,31 +2,22 @@ # layerindex-web - bulk change implementation # -# Copyright (C) 2013 Intel Corporation +# Copyright (C) 2013, 2016, 2018 Intel Corporation # # Licensed under the MIT license, see COPYING.MIT for details import sys import os -import os.path import tempfile import tarfile -import textwrap -import difflib import recipeparse import utils import shutil -from django.utils.datastructures import SortedDict logger = utils.logger_create('LayerIndexImport') -# Help us to find places to insert values -recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRC_URI', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy'] -# Variables that sometimes are a bit long but shouldn't be wrapped -nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'LIC_FILES_CHKSUM'] -meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION'] - def generate_patches(tinfoil, fetchdir, changeset, outputdir): + import oe.recipeutils tmpoutdir = tempfile.mkdtemp(dir=outputdir) last_layer = None patchname = '' @@ -50,13 +41,9 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir): outfile = open(os.path.join(tmpoutdir, patchname), 'w') last_layer = layer recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename)) - varlist = list(set(list(fields.keys()) + meta_vars)) - varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy) - filevars = localise_file_vars(recipefile, varfiles, fields.keys()) - for f, fvars in filevars.items(): - filefields = dict((k, fields[k]) for k in fvars) - patch = patch_recipe(f, layerfetchdir, filefields) - for line in patch: + patchdatalist = oe.recipeutils.patch_recipe(config_data_copy, recipefile, fields, patch=True, relpath=layerfetchdir) + for patchdata in patchdatalist: + for line in patchdata: outfile.write(line) finally: if outfile: @@ -84,113 +71,6 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir): shutil.rmtree(tmpoutdir) return ret - -def patch_recipe(fn, relpath, values): - """Update or insert variable values into a recipe file. - Note that some manual inspection/intervention may be required - since this cannot handle all situations. - """ - remainingnames = {} - for k in values.keys(): - remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1 - remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1])) - - with tempfile.NamedTemporaryFile('w', delete=False) as tf: - def outputvalue(name): - if values[name]: - rawtext = '%s = "%s"\n' % (name, values[name]) - if name in nowrap_vars: - tf.write(rawtext) - else: - wrapped = textwrap.wrap(rawtext) - for wrapline in wrapped[:-1]: - tf.write('%s \\\n' % wrapline) - tf.write('%s\n' % wrapped[-1]) - - tfn = tf.name - with open(fn, 'r') as f: - # First runthrough - find existing names (so we know not to insert based on recipe_progression) - # Second runthrough - make the changes - existingnames = [] - for runthrough in [1, 2]: - currname = None - for line in f: - if not currname: - insert = False - for k in remainingnames.keys(): - for p in recipe_progression: - if line.startswith(p): - if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames: - outputvalue(k) - del remainingnames[k] - break - for k in remainingnames.keys(): - if line.startswith(k): - currname = k - if runthrough == 1: - existingnames.append(k) - else: - del remainingnames[k] - break - if currname and runthrough > 1: - outputvalue(currname) - - if currname: - sline = line.rstrip() - if not sline.endswith('\\'): - currname = None - continue - if runthrough > 1: - tf.write(line) - f.seek(0) - if remainingnames: - tf.write('\n') - for k in remainingnames.keys(): - outputvalue(k) - - fromlines = open(fn, 'U').readlines() - tolines = open(tfn, 'U').readlines() - relfn = os.path.relpath(fn, relpath) - diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) - os.remove(tfn) - return diff - -def localise_file_vars(fn, varfiles, varlist): - from collections import defaultdict - - fndir = os.path.dirname(fn) + os.sep - - first_meta_file = None - for v in meta_vars: - f = varfiles.get(v, None) - if f: - actualdir = os.path.dirname(f) + os.sep - if actualdir.startswith(fndir): - first_meta_file = f - break - - filevars = defaultdict(list) - for v in varlist: - f = varfiles[v] - # Only return files that are in the same directory as the recipe or in some directory below there - # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable - # in if we were going to set a value specific to this recipe) - if f: - actualfile = f - else: - # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it - if first_meta_file: - actualfile = first_meta_file - else: - actualfile = fn - - actualdir = os.path.dirname(actualfile) + os.sep - if not actualdir.startswith(fndir): - actualfile = fn - filevars[actualfile].append(v) - - return filevars - def get_changeset(pk): from layerindex.models import RecipeChangeset res = list(RecipeChangeset.objects.filter(pk=pk)[:1]) @@ -229,6 +109,8 @@ def main(): sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1]) sys.exit(1) + utils.setup_core_layer_sys_path(settings, branch.name) + outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2]) finally: tinfoil.shutdown() diff --git a/layerindex/recipeparse.py b/layerindex/recipeparse.py index af0c736..f211cfa 100644 --- a/layerindex/recipeparse.py +++ b/layerindex/recipeparse.py @@ -110,20 +110,6 @@ def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch): config_data_copy.delVar('LAYERDIR') return config_data_copy -def get_var_files(fn, varlist, d): - import bb.cache - varfiles = {} - envdata = bb.cache.Cache.loadDataFull(fn, [], d) - for v in varlist: - history = envdata.varhistory.get_variable_files(v) - if history: - actualfile = history[-1] - else: - actualfile = None - varfiles[v] = actualfile - - return varfiles - machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$') distro_conf_re = re.compile(r'conf/distro/([^/.]*).conf$') bbclass_re = re.compile(r'classes/([^/.]*).bbclass$')