mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 03:49:10 +02:00
bulkchange.py: use oe.recipeutils code to patch recipes
There were several issues with this code, including that it used SortedDict which was removed in Django 1.9 and that it seemed not to have been fully updated to accommodate changes in bitbake's recipe parsing API. In the end I decided the simplest thing would be to move it over to using oe.recipeutils.patch_recipe() which is actually a now much more mature version of the code that originally started life here. With that we can get the bulk change functionality working again and gain some of the improvements in behaviour that we've developed in oe.recipeutils.patch_recipe(), as well as avoiding effectively duplicated code. Implements [YOCTO #9730]. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton
parent
7e467585ae
commit
c356f74b4e
|
|
@ -2,31 +2,22 @@
|
|||
|
||||
# layerindex-web - bulk change implementation
|
||||
#
|
||||
# Copyright (C) 2013 Intel Corporation
|
||||
# Copyright (C) 2013, 2016, 2018 Intel Corporation
|
||||
#
|
||||
# Licensed under the MIT license, see COPYING.MIT for details
|
||||
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import tempfile
|
||||
import tarfile
|
||||
import textwrap
|
||||
import difflib
|
||||
import recipeparse
|
||||
import utils
|
||||
import shutil
|
||||
from django.utils.datastructures import SortedDict
|
||||
|
||||
logger = utils.logger_create('LayerIndexImport')
|
||||
|
||||
# Help us to find places to insert values
|
||||
recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRC_URI', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
|
||||
# Variables that sometimes are a bit long but shouldn't be wrapped
|
||||
nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'LIC_FILES_CHKSUM']
|
||||
meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
|
||||
|
||||
def generate_patches(tinfoil, fetchdir, changeset, outputdir):
|
||||
import oe.recipeutils
|
||||
tmpoutdir = tempfile.mkdtemp(dir=outputdir)
|
||||
last_layer = None
|
||||
patchname = ''
|
||||
|
|
@ -50,13 +41,9 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
|
|||
outfile = open(os.path.join(tmpoutdir, patchname), 'w')
|
||||
last_layer = layer
|
||||
recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
|
||||
varlist = list(set(list(fields.keys()) + meta_vars))
|
||||
varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
|
||||
filevars = localise_file_vars(recipefile, varfiles, fields.keys())
|
||||
for f, fvars in filevars.items():
|
||||
filefields = dict((k, fields[k]) for k in fvars)
|
||||
patch = patch_recipe(f, layerfetchdir, filefields)
|
||||
for line in patch:
|
||||
patchdatalist = oe.recipeutils.patch_recipe(config_data_copy, recipefile, fields, patch=True, relpath=layerfetchdir)
|
||||
for patchdata in patchdatalist:
|
||||
for line in patchdata:
|
||||
outfile.write(line)
|
||||
finally:
|
||||
if outfile:
|
||||
|
|
@ -84,113 +71,6 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
|
|||
shutil.rmtree(tmpoutdir)
|
||||
return ret
|
||||
|
||||
|
||||
def patch_recipe(fn, relpath, values):
|
||||
"""Update or insert variable values into a recipe file.
|
||||
Note that some manual inspection/intervention may be required
|
||||
since this cannot handle all situations.
|
||||
"""
|
||||
remainingnames = {}
|
||||
for k in values.keys():
|
||||
remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
|
||||
remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
|
||||
|
||||
with tempfile.NamedTemporaryFile('w', delete=False) as tf:
|
||||
def outputvalue(name):
|
||||
if values[name]:
|
||||
rawtext = '%s = "%s"\n' % (name, values[name])
|
||||
if name in nowrap_vars:
|
||||
tf.write(rawtext)
|
||||
else:
|
||||
wrapped = textwrap.wrap(rawtext)
|
||||
for wrapline in wrapped[:-1]:
|
||||
tf.write('%s \\\n' % wrapline)
|
||||
tf.write('%s\n' % wrapped[-1])
|
||||
|
||||
tfn = tf.name
|
||||
with open(fn, 'r') as f:
|
||||
# First runthrough - find existing names (so we know not to insert based on recipe_progression)
|
||||
# Second runthrough - make the changes
|
||||
existingnames = []
|
||||
for runthrough in [1, 2]:
|
||||
currname = None
|
||||
for line in f:
|
||||
if not currname:
|
||||
insert = False
|
||||
for k in remainingnames.keys():
|
||||
for p in recipe_progression:
|
||||
if line.startswith(p):
|
||||
if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
|
||||
outputvalue(k)
|
||||
del remainingnames[k]
|
||||
break
|
||||
for k in remainingnames.keys():
|
||||
if line.startswith(k):
|
||||
currname = k
|
||||
if runthrough == 1:
|
||||
existingnames.append(k)
|
||||
else:
|
||||
del remainingnames[k]
|
||||
break
|
||||
if currname and runthrough > 1:
|
||||
outputvalue(currname)
|
||||
|
||||
if currname:
|
||||
sline = line.rstrip()
|
||||
if not sline.endswith('\\'):
|
||||
currname = None
|
||||
continue
|
||||
if runthrough > 1:
|
||||
tf.write(line)
|
||||
f.seek(0)
|
||||
if remainingnames:
|
||||
tf.write('\n')
|
||||
for k in remainingnames.keys():
|
||||
outputvalue(k)
|
||||
|
||||
fromlines = open(fn, 'U').readlines()
|
||||
tolines = open(tfn, 'U').readlines()
|
||||
relfn = os.path.relpath(fn, relpath)
|
||||
diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
|
||||
os.remove(tfn)
|
||||
return diff
|
||||
|
||||
def localise_file_vars(fn, varfiles, varlist):
|
||||
from collections import defaultdict
|
||||
|
||||
fndir = os.path.dirname(fn) + os.sep
|
||||
|
||||
first_meta_file = None
|
||||
for v in meta_vars:
|
||||
f = varfiles.get(v, None)
|
||||
if f:
|
||||
actualdir = os.path.dirname(f) + os.sep
|
||||
if actualdir.startswith(fndir):
|
||||
first_meta_file = f
|
||||
break
|
||||
|
||||
filevars = defaultdict(list)
|
||||
for v in varlist:
|
||||
f = varfiles[v]
|
||||
# Only return files that are in the same directory as the recipe or in some directory below there
|
||||
# (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
|
||||
# in if we were going to set a value specific to this recipe)
|
||||
if f:
|
||||
actualfile = f
|
||||
else:
|
||||
# Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
|
||||
if first_meta_file:
|
||||
actualfile = first_meta_file
|
||||
else:
|
||||
actualfile = fn
|
||||
|
||||
actualdir = os.path.dirname(actualfile) + os.sep
|
||||
if not actualdir.startswith(fndir):
|
||||
actualfile = fn
|
||||
filevars[actualfile].append(v)
|
||||
|
||||
return filevars
|
||||
|
||||
def get_changeset(pk):
|
||||
from layerindex.models import RecipeChangeset
|
||||
res = list(RecipeChangeset.objects.filter(pk=pk)[:1])
|
||||
|
|
@ -229,6 +109,8 @@ def main():
|
|||
sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1])
|
||||
sys.exit(1)
|
||||
|
||||
utils.setup_core_layer_sys_path(settings, branch.name)
|
||||
|
||||
outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
|
||||
finally:
|
||||
tinfoil.shutdown()
|
||||
|
|
|
|||
|
|
@ -110,20 +110,6 @@ def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch):
|
|||
config_data_copy.delVar('LAYERDIR')
|
||||
return config_data_copy
|
||||
|
||||
def get_var_files(fn, varlist, d):
|
||||
import bb.cache
|
||||
varfiles = {}
|
||||
envdata = bb.cache.Cache.loadDataFull(fn, [], d)
|
||||
for v in varlist:
|
||||
history = envdata.varhistory.get_variable_files(v)
|
||||
if history:
|
||||
actualfile = history[-1]
|
||||
else:
|
||||
actualfile = None
|
||||
varfiles[v] = actualfile
|
||||
|
||||
return varfiles
|
||||
|
||||
machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
|
||||
distro_conf_re = re.compile(r'conf/distro/([^/.]*).conf$')
|
||||
bbclass_re = re.compile(r'classes/([^/.]*).bbclass$')
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user