mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
f2e5d88461
If a line like:
foo=${@' '.join([d.getVar('D', True) + x for x in (' '.join([d.getVar('FILES_bash-' + p, True) or '' for p in ['lib', 'dev', 'staticdev', 'doc', 'locale', 'ptest']])).split()])}
is added to a function like do_install, it fails with Exception name 'd'
is not defined. This is due to a change of behaviour in python 3 compared
to python 2. Generator expressions, dict comprehensions and set comprehensions
are executed in a new scope but list comprehensions in python 2.x are not. In
python 3 they all use a new scope.
To allow these kinds of expressions to work, the easiest approach is
to add 'd' to the global context. To do this, an extra optional parameter
is added to better_eval and we use that to add 'd'.
(Bitbake rev: 8f74881037bb01013d3d439dc0c269909a198c1c)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
969 lines
34 KiB
Python
969 lines
34 KiB
Python
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
"""
|
|
BitBake Smart Dictionary Implementation
|
|
|
|
Functions for interacting with the data structure used by the
|
|
BitBake build tools.
|
|
|
|
"""
|
|
|
|
# Copyright (C) 2003, 2004 Chris Larson
|
|
# Copyright (C) 2004, 2005 Seb Frankengul
|
|
# Copyright (C) 2005, 2006 Holger Hans Peter Freyther
|
|
# Copyright (C) 2005 Uli Luckas
|
|
# Copyright (C) 2005 ROAD GmbH
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
# Based on functions from the base bb module, Copyright 2003 Holger Schurig
|
|
|
|
import copy, re, sys, traceback
|
|
from collections import MutableMapping
|
|
import logging
|
|
import hashlib
|
|
import bb, bb.codeparser
|
|
from bb import utils
|
|
from bb.COW import COWDictBase
|
|
|
|
logger = logging.getLogger("BitBake.Data")
|
|
|
|
__setvar_keyword__ = ["_append", "_prepend", "_remove"]
|
|
__setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>.*))?$')
|
|
__expand_var_regexp__ = re.compile(r"\${[^{}@\n\t :]+}")
|
|
__expand_python_regexp__ = re.compile(r"\${@.+?}")
|
|
|
|
def infer_caller_details(loginfo, parent = False, varval = True):
|
|
"""Save the caller the trouble of specifying everything."""
|
|
# Save effort.
|
|
if 'ignore' in loginfo and loginfo['ignore']:
|
|
return
|
|
# If nothing was provided, mark this as possibly unneeded.
|
|
if not loginfo:
|
|
loginfo['ignore'] = True
|
|
return
|
|
# Infer caller's likely values for variable (var) and value (value),
|
|
# to reduce clutter in the rest of the code.
|
|
above = None
|
|
def set_above():
|
|
try:
|
|
raise Exception
|
|
except Exception:
|
|
tb = sys.exc_info()[2]
|
|
if parent:
|
|
return tb.tb_frame.f_back.f_back.f_back
|
|
else:
|
|
return tb.tb_frame.f_back.f_back
|
|
|
|
if varval and ('variable' not in loginfo or 'detail' not in loginfo):
|
|
if not above:
|
|
above = set_above()
|
|
lcls = above.f_locals.items()
|
|
for k, v in lcls:
|
|
if k == 'value' and 'detail' not in loginfo:
|
|
loginfo['detail'] = v
|
|
if k == 'var' and 'variable' not in loginfo:
|
|
loginfo['variable'] = v
|
|
# Infer file/line/function from traceback
|
|
# Don't use traceback.extract_stack() since it fills the line contents which
|
|
# we don't need and that hits stat syscalls
|
|
if 'file' not in loginfo:
|
|
if not above:
|
|
above = set_above()
|
|
f = above.f_back
|
|
line = f.f_lineno
|
|
file = f.f_code.co_filename
|
|
func = f.f_code.co_name
|
|
loginfo['file'] = file
|
|
loginfo['line'] = line
|
|
if func not in loginfo:
|
|
loginfo['func'] = func
|
|
|
|
class VariableParse:
|
|
def __init__(self, varname, d, val = None):
|
|
self.varname = varname
|
|
self.d = d
|
|
self.value = val
|
|
|
|
self.references = set()
|
|
self.execs = set()
|
|
self.contains = {}
|
|
|
|
def var_sub(self, match):
|
|
key = match.group()[2:-1]
|
|
if self.varname and key:
|
|
if self.varname == key:
|
|
raise Exception("variable %s references itself!" % self.varname)
|
|
if key in self.d.expand_cache:
|
|
varparse = self.d.expand_cache[key]
|
|
var = varparse.value
|
|
else:
|
|
var = self.d.getVarFlag(key, "_content", True)
|
|
self.references.add(key)
|
|
if var is not None:
|
|
return var
|
|
else:
|
|
return match.group()
|
|
|
|
def python_sub(self, match):
|
|
code = match.group()[3:-1]
|
|
codeobj = compile(code.strip(), self.varname or "<expansion>", "eval")
|
|
|
|
parser = bb.codeparser.PythonParser(self.varname, logger)
|
|
parser.parse_python(code)
|
|
if self.varname:
|
|
vardeps = self.d.getVarFlag(self.varname, "vardeps", True)
|
|
if vardeps is None:
|
|
parser.log.flush()
|
|
else:
|
|
parser.log.flush()
|
|
self.references |= parser.references
|
|
self.execs |= parser.execs
|
|
|
|
for k in parser.contains:
|
|
if k not in self.contains:
|
|
self.contains[k] = parser.contains[k].copy()
|
|
else:
|
|
self.contains[k].update(parser.contains[k])
|
|
value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
|
|
return str(value)
|
|
|
|
|
|
class DataContext(dict):
|
|
def __init__(self, metadata, **kwargs):
|
|
self.metadata = metadata
|
|
dict.__init__(self, **kwargs)
|
|
self['d'] = metadata
|
|
|
|
def __missing__(self, key):
|
|
value = self.metadata.getVar(key, True)
|
|
if value is None or self.metadata.getVarFlag(key, 'func', False):
|
|
raise KeyError(key)
|
|
else:
|
|
return value
|
|
|
|
class ExpansionError(Exception):
|
|
def __init__(self, varname, expression, exception):
|
|
self.expression = expression
|
|
self.variablename = varname
|
|
self.exception = exception
|
|
if varname:
|
|
if expression:
|
|
self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
|
|
else:
|
|
self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception)
|
|
else:
|
|
self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
|
|
Exception.__init__(self, self.msg)
|
|
self.args = (varname, expression, exception)
|
|
def __str__(self):
|
|
return self.msg
|
|
|
|
class IncludeHistory(object):
|
|
def __init__(self, parent = None, filename = '[TOP LEVEL]'):
|
|
self.parent = parent
|
|
self.filename = filename
|
|
self.children = []
|
|
self.current = self
|
|
|
|
def copy(self):
|
|
new = IncludeHistory(self.parent, self.filename)
|
|
for c in self.children:
|
|
new.children.append(c)
|
|
return new
|
|
|
|
def include(self, filename):
|
|
newfile = IncludeHistory(self.current, filename)
|
|
self.current.children.append(newfile)
|
|
self.current = newfile
|
|
return self
|
|
|
|
def __enter__(self):
|
|
pass
|
|
|
|
def __exit__(self, a, b, c):
|
|
if self.current.parent:
|
|
self.current = self.current.parent
|
|
else:
|
|
bb.warn("Include log: Tried to finish '%s' at top level." % filename)
|
|
return False
|
|
|
|
def emit(self, o, level = 0):
|
|
"""Emit an include history file, and its children."""
|
|
if level:
|
|
spaces = " " * (level - 1)
|
|
o.write("# %s%s" % (spaces, self.filename))
|
|
if len(self.children) > 0:
|
|
o.write(" includes:")
|
|
else:
|
|
o.write("#\n# INCLUDE HISTORY:\n#")
|
|
level = level + 1
|
|
for child in self.children:
|
|
o.write("\n")
|
|
child.emit(o, level)
|
|
|
|
class VariableHistory(object):
|
|
def __init__(self, dataroot):
|
|
self.dataroot = dataroot
|
|
self.variables = COWDictBase.copy()
|
|
|
|
def copy(self):
|
|
new = VariableHistory(self.dataroot)
|
|
new.variables = self.variables.copy()
|
|
return new
|
|
|
|
def record(self, *kwonly, **loginfo):
|
|
if not self.dataroot._tracking:
|
|
return
|
|
if len(kwonly) > 0:
|
|
raise TypeError
|
|
infer_caller_details(loginfo, parent = True)
|
|
if 'ignore' in loginfo and loginfo['ignore']:
|
|
return
|
|
if 'op' not in loginfo or not loginfo['op']:
|
|
loginfo['op'] = 'set'
|
|
if 'detail' in loginfo:
|
|
loginfo['detail'] = str(loginfo['detail'])
|
|
if 'variable' not in loginfo or 'file' not in loginfo:
|
|
raise ValueError("record() missing variable or file.")
|
|
var = loginfo['variable']
|
|
|
|
if var not in self.variables:
|
|
self.variables[var] = []
|
|
if not isinstance(self.variables[var], list):
|
|
return
|
|
if 'nodups' in loginfo and loginfo in self.variables[var]:
|
|
return
|
|
self.variables[var].append(loginfo.copy())
|
|
|
|
def variable(self, var):
|
|
if var in self.variables:
|
|
return self.variables[var]
|
|
else:
|
|
return []
|
|
|
|
def emit(self, var, oval, val, o, d):
|
|
history = self.variable(var)
|
|
|
|
# Append override history
|
|
if var in d.overridedata:
|
|
for (r, override) in d.overridedata[var]:
|
|
for event in self.variable(r):
|
|
loginfo = event.copy()
|
|
if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
|
|
continue
|
|
loginfo['variable'] = var
|
|
loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
|
|
history.append(loginfo)
|
|
|
|
commentVal = re.sub('\n', '\n#', str(oval))
|
|
if history:
|
|
if len(history) == 1:
|
|
o.write("#\n# $%s\n" % var)
|
|
else:
|
|
o.write("#\n# $%s [%d operations]\n" % (var, len(history)))
|
|
for event in history:
|
|
# o.write("# %s\n" % str(event))
|
|
if 'func' in event:
|
|
# If we have a function listed, this is internal
|
|
# code, not an operation in a config file, and the
|
|
# full path is distracting.
|
|
event['file'] = re.sub('.*/', '', event['file'])
|
|
display_func = ' [%s]' % event['func']
|
|
else:
|
|
display_func = ''
|
|
if 'flag' in event:
|
|
flag = '[%s] ' % (event['flag'])
|
|
else:
|
|
flag = ''
|
|
o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail'])))
|
|
if len(history) > 1:
|
|
o.write("# pre-expansion value:\n")
|
|
o.write('# "%s"\n' % (commentVal))
|
|
else:
|
|
o.write("#\n# $%s\n# [no history recorded]\n#\n" % var)
|
|
o.write('# "%s"\n' % (commentVal))
|
|
|
|
def get_variable_files(self, var):
|
|
"""Get the files where operations are made on a variable"""
|
|
var_history = self.variable(var)
|
|
files = []
|
|
for event in var_history:
|
|
files.append(event['file'])
|
|
return files
|
|
|
|
def get_variable_lines(self, var, f):
|
|
"""Get the line where a operation is made on a variable in file f"""
|
|
var_history = self.variable(var)
|
|
lines = []
|
|
for event in var_history:
|
|
if f== event['file']:
|
|
line = event['line']
|
|
lines.append(line)
|
|
return lines
|
|
|
|
def get_variable_items_files(self, var, d):
|
|
"""
|
|
Use variable history to map items added to a list variable and
|
|
the files in which they were added.
|
|
"""
|
|
history = self.variable(var)
|
|
finalitems = (d.getVar(var, True) or '').split()
|
|
filemap = {}
|
|
isset = False
|
|
for event in history:
|
|
if 'flag' in event:
|
|
continue
|
|
if event['op'] == '_remove':
|
|
continue
|
|
if isset and event['op'] == 'set?':
|
|
continue
|
|
isset = True
|
|
items = d.expand(event['detail']).split()
|
|
for item in items:
|
|
# This is a little crude but is belt-and-braces to avoid us
|
|
# having to handle every possible operation type specifically
|
|
if item in finalitems and not item in filemap:
|
|
filemap[item] = event['file']
|
|
return filemap
|
|
|
|
def del_var_history(self, var, f=None, line=None):
|
|
"""If file f and line are not given, the entire history of var is deleted"""
|
|
if var in self.variables:
|
|
if f and line:
|
|
self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line]
|
|
else:
|
|
self.variables[var] = []
|
|
|
|
class DataSmart(MutableMapping):
|
|
def __init__(self):
|
|
self.dict = {}
|
|
|
|
self.inchistory = IncludeHistory()
|
|
self.varhistory = VariableHistory(self)
|
|
self._tracking = False
|
|
|
|
self.expand_cache = {}
|
|
|
|
# cookie monster tribute
|
|
# Need to be careful about writes to overridedata as
|
|
# its only a shallow copy, could influence other data store
|
|
# copies!
|
|
self.overridedata = {}
|
|
self.overrides = None
|
|
self.overridevars = set(["OVERRIDES", "FILE"])
|
|
self.inoverride = False
|
|
|
|
def enableTracking(self):
|
|
self._tracking = True
|
|
|
|
def disableTracking(self):
|
|
self._tracking = False
|
|
|
|
def expandWithRefs(self, s, varname):
|
|
|
|
if not isinstance(s, str): # sanity check
|
|
return VariableParse(varname, self, s)
|
|
|
|
if varname and varname in self.expand_cache:
|
|
return self.expand_cache[varname]
|
|
|
|
varparse = VariableParse(varname, self)
|
|
|
|
while s.find('${') != -1:
|
|
olds = s
|
|
try:
|
|
s = __expand_var_regexp__.sub(varparse.var_sub, s)
|
|
try:
|
|
s = __expand_python_regexp__.sub(varparse.python_sub, s)
|
|
except SyntaxError as e:
|
|
# Likely unmatched brackets, just don't expand the expression
|
|
if e.msg != "EOL while scanning string literal":
|
|
raise
|
|
if s == olds:
|
|
break
|
|
except ExpansionError:
|
|
raise
|
|
except bb.parse.SkipRecipe:
|
|
raise
|
|
except Exception as exc:
|
|
raise ExpansionError(varname, s, exc) from exc
|
|
|
|
varparse.value = s
|
|
|
|
if varname:
|
|
self.expand_cache[varname] = varparse
|
|
|
|
return varparse
|
|
|
|
def expand(self, s, varname = None):
|
|
return self.expandWithRefs(s, varname).value
|
|
|
|
def finalize(self, parent = False):
|
|
return
|
|
|
|
def internal_finalize(self, parent = False):
|
|
"""Performs final steps upon the datastore, including application of overrides"""
|
|
self.overrides = None
|
|
|
|
def need_overrides(self):
|
|
if self.overrides is not None:
|
|
return
|
|
if self.inoverride:
|
|
return
|
|
for count in range(5):
|
|
self.inoverride = True
|
|
# Can end up here recursively so setup dummy values
|
|
self.overrides = []
|
|
self.overridesset = set()
|
|
self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
|
|
self.overridesset = set(self.overrides)
|
|
self.inoverride = False
|
|
self.expand_cache = {}
|
|
newoverrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
|
|
if newoverrides == self.overrides:
|
|
break
|
|
self.overrides = newoverrides
|
|
self.overridesset = set(self.overrides)
|
|
else:
|
|
bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
|
|
|
|
def initVar(self, var):
|
|
self.expand_cache = {}
|
|
if not var in self.dict:
|
|
self.dict[var] = {}
|
|
|
|
def _findVar(self, var):
|
|
dest = self.dict
|
|
while dest:
|
|
if var in dest:
|
|
return dest[var]
|
|
|
|
if "_data" not in dest:
|
|
break
|
|
dest = dest["_data"]
|
|
|
|
def _makeShadowCopy(self, var):
|
|
if var in self.dict:
|
|
return
|
|
|
|
local_var = self._findVar(var)
|
|
|
|
if local_var:
|
|
self.dict[var] = copy.copy(local_var)
|
|
else:
|
|
self.initVar(var)
|
|
|
|
|
|
def setVar(self, var, value, **loginfo):
|
|
#print("var=" + str(var) + " val=" + str(value))
|
|
parsing=False
|
|
if 'parsing' in loginfo:
|
|
parsing=True
|
|
|
|
if 'op' not in loginfo:
|
|
loginfo['op'] = "set"
|
|
self.expand_cache = {}
|
|
match = __setvar_regexp__.match(var)
|
|
if match and match.group("keyword") in __setvar_keyword__:
|
|
base = match.group('base')
|
|
keyword = match.group("keyword")
|
|
override = match.group('add')
|
|
l = self.getVarFlag(base, keyword, False) or []
|
|
l.append([value, override])
|
|
self.setVarFlag(base, keyword, l, ignore=True)
|
|
# And cause that to be recorded:
|
|
loginfo['detail'] = value
|
|
loginfo['variable'] = base
|
|
if override:
|
|
loginfo['op'] = '%s[%s]' % (keyword, override)
|
|
else:
|
|
loginfo['op'] = keyword
|
|
self.varhistory.record(**loginfo)
|
|
# todo make sure keyword is not __doc__ or __module__
|
|
# pay the cookie monster
|
|
|
|
# more cookies for the cookie monster
|
|
if '_' in var:
|
|
self._setvar_update_overrides(base, **loginfo)
|
|
|
|
if base in self.overridevars:
|
|
self._setvar_update_overridevars(var, value)
|
|
return
|
|
|
|
if not var in self.dict:
|
|
self._makeShadowCopy(var)
|
|
|
|
if not parsing:
|
|
if "_append" in self.dict[var]:
|
|
del self.dict[var]["_append"]
|
|
if "_prepend" in self.dict[var]:
|
|
del self.dict[var]["_prepend"]
|
|
if var in self.overridedata:
|
|
active = []
|
|
self.need_overrides()
|
|
for (r, o) in self.overridedata[var]:
|
|
if o in self.overridesset:
|
|
active.append(r)
|
|
elif "_" in o:
|
|
if set(o.split("_")).issubset(self.overridesset):
|
|
active.append(r)
|
|
for a in active:
|
|
self.delVar(a)
|
|
del self.overridedata[var]
|
|
|
|
# more cookies for the cookie monster
|
|
if '_' in var:
|
|
self._setvar_update_overrides(var, **loginfo)
|
|
|
|
# setting var
|
|
self.dict[var]["_content"] = value
|
|
self.varhistory.record(**loginfo)
|
|
|
|
if var in self.overridevars:
|
|
self._setvar_update_overridevars(var, value)
|
|
|
|
def _setvar_update_overridevars(self, var, value):
|
|
vardata = self.expandWithRefs(value, var)
|
|
new = vardata.references
|
|
new.update(vardata.contains.keys())
|
|
while not new.issubset(self.overridevars):
|
|
nextnew = set()
|
|
self.overridevars.update(new)
|
|
for i in new:
|
|
vardata = self.expandWithRefs(self.getVar(i, True), i)
|
|
nextnew.update(vardata.references)
|
|
nextnew.update(vardata.contains.keys())
|
|
new = nextnew
|
|
self.internal_finalize(True)
|
|
|
|
def _setvar_update_overrides(self, var, **loginfo):
|
|
# aka pay the cookie monster
|
|
override = var[var.rfind('_')+1:]
|
|
shortvar = var[:var.rfind('_')]
|
|
while override and override.islower():
|
|
if shortvar not in self.overridedata:
|
|
self.overridedata[shortvar] = []
|
|
if [var, override] not in self.overridedata[shortvar]:
|
|
# Force CoW by recreating the list first
|
|
self.overridedata[shortvar] = list(self.overridedata[shortvar])
|
|
self.overridedata[shortvar].append([var, override])
|
|
override = None
|
|
if "_" in shortvar:
|
|
override = var[shortvar.rfind('_')+1:]
|
|
shortvar = var[:shortvar.rfind('_')]
|
|
if len(shortvar) == 0:
|
|
override = None
|
|
|
|
def getVar(self, var, expand, noweakdefault=False, parsing=False):
|
|
return self.getVarFlag(var, "_content", expand, noweakdefault, parsing)
|
|
|
|
def renameVar(self, key, newkey, **loginfo):
|
|
"""
|
|
Rename the variable key to newkey
|
|
"""
|
|
val = self.getVar(key, 0, parsing=True)
|
|
if val is not None:
|
|
loginfo['variable'] = newkey
|
|
loginfo['op'] = 'rename from %s' % key
|
|
loginfo['detail'] = val
|
|
self.varhistory.record(**loginfo)
|
|
self.setVar(newkey, val, ignore=True, parsing=True)
|
|
|
|
for i in (__setvar_keyword__):
|
|
src = self.getVarFlag(key, i, False)
|
|
if src is None:
|
|
continue
|
|
|
|
dest = self.getVarFlag(newkey, i, False) or []
|
|
dest.extend(src)
|
|
self.setVarFlag(newkey, i, dest, ignore=True)
|
|
|
|
if key in self.overridedata:
|
|
self.overridedata[newkey] = []
|
|
for (v, o) in self.overridedata[key]:
|
|
self.overridedata[newkey].append([v.replace(key, newkey), o])
|
|
self.renameVar(v, v.replace(key, newkey))
|
|
|
|
if '_' in newkey and val is None:
|
|
self._setvar_update_overrides(newkey, **loginfo)
|
|
|
|
loginfo['variable'] = key
|
|
loginfo['op'] = 'rename (to)'
|
|
loginfo['detail'] = newkey
|
|
self.varhistory.record(**loginfo)
|
|
self.delVar(key, ignore=True)
|
|
|
|
def appendVar(self, var, value, **loginfo):
|
|
loginfo['op'] = 'append'
|
|
self.varhistory.record(**loginfo)
|
|
self.setVar(var + "_append", value, ignore=True, parsing=True)
|
|
|
|
def prependVar(self, var, value, **loginfo):
|
|
loginfo['op'] = 'prepend'
|
|
self.varhistory.record(**loginfo)
|
|
self.setVar(var + "_prepend", value, ignore=True, parsing=True)
|
|
|
|
def delVar(self, var, **loginfo):
|
|
loginfo['detail'] = ""
|
|
loginfo['op'] = 'del'
|
|
self.varhistory.record(**loginfo)
|
|
self.expand_cache = {}
|
|
self.dict[var] = {}
|
|
if var in self.overridedata:
|
|
del self.overridedata[var]
|
|
if '_' in var:
|
|
override = var[var.rfind('_')+1:]
|
|
shortvar = var[:var.rfind('_')]
|
|
while override and override.islower():
|
|
try:
|
|
if shortvar in self.overridedata:
|
|
# Force CoW by recreating the list first
|
|
self.overridedata[shortvar] = list(self.overridedata[shortvar])
|
|
self.overridedata[shortvar].remove([var, override])
|
|
except ValueError as e:
|
|
pass
|
|
override = None
|
|
if "_" in shortvar:
|
|
override = var[shortvar.rfind('_')+1:]
|
|
shortvar = var[:shortvar.rfind('_')]
|
|
if len(shortvar) == 0:
|
|
override = None
|
|
|
|
def setVarFlag(self, var, flag, value, **loginfo):
|
|
self.expand_cache = {}
|
|
if 'op' not in loginfo:
|
|
loginfo['op'] = "set"
|
|
loginfo['flag'] = flag
|
|
self.varhistory.record(**loginfo)
|
|
if not var in self.dict:
|
|
self._makeShadowCopy(var)
|
|
self.dict[var][flag] = value
|
|
|
|
if flag == "_defaultval" and '_' in var:
|
|
self._setvar_update_overrides(var, **loginfo)
|
|
if flag == "_defaultval" and var in self.overridevars:
|
|
self._setvar_update_overridevars(var, value)
|
|
|
|
if flag == "unexport" or flag == "export":
|
|
if not "__exportlist" in self.dict:
|
|
self._makeShadowCopy("__exportlist")
|
|
if not "_content" in self.dict["__exportlist"]:
|
|
self.dict["__exportlist"]["_content"] = set()
|
|
self.dict["__exportlist"]["_content"].add(var)
|
|
|
|
def getVarFlag(self, var, flag, expand, noweakdefault=False, parsing=False):
|
|
local_var = self._findVar(var)
|
|
value = None
|
|
if flag == "_content" and var in self.overridedata and not parsing:
|
|
match = False
|
|
active = {}
|
|
self.need_overrides()
|
|
for (r, o) in self.overridedata[var]:
|
|
# What about double overrides both with "_" in the name?
|
|
if o in self.overridesset:
|
|
active[o] = r
|
|
elif "_" in o:
|
|
if set(o.split("_")).issubset(self.overridesset):
|
|
active[o] = r
|
|
|
|
mod = True
|
|
while mod:
|
|
mod = False
|
|
for o in self.overrides:
|
|
for a in active.copy():
|
|
if a.endswith("_" + o):
|
|
t = active[a]
|
|
del active[a]
|
|
active[a.replace("_" + o, "")] = t
|
|
mod = True
|
|
elif a == o:
|
|
match = active[a]
|
|
del active[a]
|
|
if match:
|
|
value = self.getVar(match, False)
|
|
|
|
if local_var is not None and value is None:
|
|
if flag in local_var:
|
|
value = copy.copy(local_var[flag])
|
|
elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
|
|
value = copy.copy(local_var["_defaultval"])
|
|
|
|
|
|
if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
|
|
if not value:
|
|
value = ""
|
|
self.need_overrides()
|
|
for (r, o) in local_var["_append"]:
|
|
match = True
|
|
if o:
|
|
for o2 in o.split("_"):
|
|
if not o2 in self.overrides:
|
|
match = False
|
|
if match:
|
|
value = value + r
|
|
|
|
if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
|
|
if not value:
|
|
value = ""
|
|
self.need_overrides()
|
|
for (r, o) in local_var["_prepend"]:
|
|
|
|
match = True
|
|
if o:
|
|
for o2 in o.split("_"):
|
|
if not o2 in self.overrides:
|
|
match = False
|
|
if match:
|
|
value = r + value
|
|
|
|
if expand and value:
|
|
# Only getvar (flag == _content) hits the expand cache
|
|
cachename = None
|
|
if flag == "_content":
|
|
cachename = var
|
|
else:
|
|
cachename = var + "[" + flag + "]"
|
|
value = self.expand(value, cachename)
|
|
|
|
if value and flag == "_content" and local_var is not None and "_remove" in local_var:
|
|
removes = []
|
|
self.need_overrides()
|
|
for (r, o) in local_var["_remove"]:
|
|
match = True
|
|
if o:
|
|
for o2 in o.split("_"):
|
|
if not o2 in self.overrides:
|
|
match = False
|
|
if match:
|
|
removes.extend(self.expand(r).split())
|
|
|
|
filtered = filter(lambda v: v not in removes,
|
|
value.split())
|
|
value = " ".join(filtered)
|
|
if expand and var in self.expand_cache:
|
|
# We need to ensure the expand cache has the correct value
|
|
# flag == "_content" here
|
|
self.expand_cache[var].value = value
|
|
return value
|
|
|
|
def delVarFlag(self, var, flag, **loginfo):
|
|
self.expand_cache = {}
|
|
local_var = self._findVar(var)
|
|
if not local_var:
|
|
return
|
|
if not var in self.dict:
|
|
self._makeShadowCopy(var)
|
|
|
|
if var in self.dict and flag in self.dict[var]:
|
|
loginfo['detail'] = ""
|
|
loginfo['op'] = 'delFlag'
|
|
loginfo['flag'] = flag
|
|
self.varhistory.record(**loginfo)
|
|
|
|
del self.dict[var][flag]
|
|
|
|
def appendVarFlag(self, var, flag, value, **loginfo):
|
|
loginfo['op'] = 'append'
|
|
loginfo['flag'] = flag
|
|
self.varhistory.record(**loginfo)
|
|
newvalue = (self.getVarFlag(var, flag, False) or "") + value
|
|
self.setVarFlag(var, flag, newvalue, ignore=True)
|
|
|
|
def prependVarFlag(self, var, flag, value, **loginfo):
|
|
loginfo['op'] = 'prepend'
|
|
loginfo['flag'] = flag
|
|
self.varhistory.record(**loginfo)
|
|
newvalue = value + (self.getVarFlag(var, flag, False) or "")
|
|
self.setVarFlag(var, flag, newvalue, ignore=True)
|
|
|
|
def setVarFlags(self, var, flags, **loginfo):
|
|
self.expand_cache = {}
|
|
infer_caller_details(loginfo)
|
|
if not var in self.dict:
|
|
self._makeShadowCopy(var)
|
|
|
|
for i in flags:
|
|
if i == "_content":
|
|
continue
|
|
loginfo['flag'] = i
|
|
loginfo['detail'] = flags[i]
|
|
self.varhistory.record(**loginfo)
|
|
self.dict[var][i] = flags[i]
|
|
|
|
def getVarFlags(self, var, expand = False, internalflags=False):
|
|
local_var = self._findVar(var)
|
|
flags = {}
|
|
|
|
if local_var:
|
|
for i in local_var:
|
|
if i.startswith("_") and not internalflags:
|
|
continue
|
|
flags[i] = local_var[i]
|
|
if expand and i in expand:
|
|
flags[i] = self.expand(flags[i], var + "[" + i + "]")
|
|
if len(flags) == 0:
|
|
return None
|
|
return flags
|
|
|
|
|
|
def delVarFlags(self, var, **loginfo):
|
|
self.expand_cache = {}
|
|
if not var in self.dict:
|
|
self._makeShadowCopy(var)
|
|
|
|
if var in self.dict:
|
|
content = None
|
|
|
|
loginfo['op'] = 'delete flags'
|
|
self.varhistory.record(**loginfo)
|
|
|
|
# try to save the content
|
|
if "_content" in self.dict[var]:
|
|
content = self.dict[var]["_content"]
|
|
self.dict[var] = {}
|
|
self.dict[var]["_content"] = content
|
|
else:
|
|
del self.dict[var]
|
|
|
|
def createCopy(self):
|
|
"""
|
|
Create a copy of self by setting _data to self
|
|
"""
|
|
# we really want this to be a DataSmart...
|
|
data = DataSmart()
|
|
data.dict["_data"] = self.dict
|
|
data.varhistory = self.varhistory.copy()
|
|
data.varhistory.datasmart = data
|
|
data.inchistory = self.inchistory.copy()
|
|
|
|
data._tracking = self._tracking
|
|
|
|
data.overrides = None
|
|
data.overridevars = copy.copy(self.overridevars)
|
|
# Should really be a deepcopy but has heavy overhead.
|
|
# Instead, we're careful with writes.
|
|
data.overridedata = copy.copy(self.overridedata)
|
|
|
|
return data
|
|
|
|
def expandVarref(self, variable, parents=False):
|
|
"""Find all references to variable in the data and expand it
|
|
in place, optionally descending to parent datastores."""
|
|
|
|
if parents:
|
|
keys = iter(self)
|
|
else:
|
|
keys = self.localkeys()
|
|
|
|
ref = '${%s}' % variable
|
|
value = self.getVar(variable, False)
|
|
for key in keys:
|
|
referrervalue = self.getVar(key, False)
|
|
if referrervalue and ref in referrervalue:
|
|
self.setVar(key, referrervalue.replace(ref, value))
|
|
|
|
def localkeys(self):
|
|
for key in self.dict:
|
|
if key != '_data':
|
|
yield key
|
|
|
|
def __iter__(self):
|
|
deleted = set()
|
|
overrides = set()
|
|
def keylist(d):
|
|
klist = set()
|
|
for key in d:
|
|
if key == "_data":
|
|
continue
|
|
if key in deleted:
|
|
continue
|
|
if key in overrides:
|
|
continue
|
|
if not d[key]:
|
|
deleted.add(key)
|
|
continue
|
|
klist.add(key)
|
|
|
|
if "_data" in d:
|
|
klist |= keylist(d["_data"])
|
|
|
|
return klist
|
|
|
|
self.need_overrides()
|
|
for var in self.overridedata:
|
|
for (r, o) in self.overridedata[var]:
|
|
if o in self.overridesset:
|
|
overrides.add(var)
|
|
elif "_" in o:
|
|
if set(o.split("_")).issubset(self.overridesset):
|
|
overrides.add(var)
|
|
|
|
for k in keylist(self.dict):
|
|
yield k
|
|
|
|
for k in overrides:
|
|
yield k
|
|
|
|
def __len__(self):
|
|
return len(frozenset(iter(self)))
|
|
|
|
def __getitem__(self, item):
|
|
value = self.getVar(item, False)
|
|
if value is None:
|
|
raise KeyError(item)
|
|
else:
|
|
return value
|
|
|
|
def __setitem__(self, var, value):
|
|
self.setVar(var, value)
|
|
|
|
def __delitem__(self, var):
|
|
self.delVar(var)
|
|
|
|
def get_hash(self):
|
|
data = {}
|
|
d = self.createCopy()
|
|
bb.data.expandKeys(d)
|
|
bb.data.update_data(d)
|
|
|
|
config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split())
|
|
keys = set(key for key in iter(d) if not key.startswith("__"))
|
|
for key in keys:
|
|
if key in config_whitelist:
|
|
continue
|
|
|
|
value = d.getVar(key, False) or ""
|
|
data.update({key:value})
|
|
|
|
varflags = d.getVarFlags(key, internalflags = True)
|
|
if not varflags:
|
|
continue
|
|
for f in varflags:
|
|
if f == "_content":
|
|
continue
|
|
data.update({'%s[%s]' % (key, f):varflags[f]})
|
|
|
|
for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]:
|
|
bb_list = d.getVar(key, False) or []
|
|
bb_list.sort()
|
|
data.update({key:str(bb_list)})
|
|
|
|
if key == "__BBANONFUNCS":
|
|
for i in bb_list:
|
|
value = d.getVar(i, False) or ""
|
|
data.update({i:value})
|
|
|
|
data_str = str([(k, data[k]) for k in sorted(data.keys())])
|
|
return hashlib.md5(data_str.encode("utf-8")).hexdigest()
|