mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 12:59:02 +02:00
recipetool/create_buildsys_python: refactor code for futur PEP517 addition
In order to prepare the support for pyproject.toml (PEP517 [1]) enabled projects, refactor the code and move setup.py specific code into a specific class in order to allow sharing the PythonRecipeHandler class No functionnal changes expected [1]: https://peps.python.org/pep-0517/#source-tree (From OE-Core rev: 2281e93347da4129062cfb40710df03c87c63168) Signed-off-by: Julien Stephan <jstephan@baylibre.com> Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Julien Stephan
Richard Purdie
parent
be129bd0bc
commit
e64e92f2de
|
|
@ -37,63 +37,8 @@ class PythonRecipeHandler(RecipeHandler):
|
|||
assume_provided = ['builtins', 'os.path']
|
||||
# Assumes that the host python3 builtin_module_names is sane for target too
|
||||
assume_provided = assume_provided + list(sys.builtin_module_names)
|
||||
excluded_fields = []
|
||||
|
||||
bbvar_map = {
|
||||
'Name': 'PN',
|
||||
'Version': 'PV',
|
||||
'Home-page': 'HOMEPAGE',
|
||||
'Summary': 'SUMMARY',
|
||||
'Description': 'DESCRIPTION',
|
||||
'License': 'LICENSE',
|
||||
'Requires': 'RDEPENDS:${PN}',
|
||||
'Provides': 'RPROVIDES:${PN}',
|
||||
'Obsoletes': 'RREPLACES:${PN}',
|
||||
}
|
||||
# PN/PV are already set by recipetool core & desc can be extremely long
|
||||
excluded_fields = [
|
||||
'Description',
|
||||
]
|
||||
setup_parse_map = {
|
||||
'Url': 'Home-page',
|
||||
'Classifiers': 'Classifier',
|
||||
'Description': 'Summary',
|
||||
}
|
||||
setuparg_map = {
|
||||
'Home-page': 'url',
|
||||
'Classifier': 'classifiers',
|
||||
'Summary': 'description',
|
||||
'Description': 'long-description',
|
||||
}
|
||||
# Values which are lists, used by the setup.py argument based metadata
|
||||
# extraction method, to determine how to process the setup.py output.
|
||||
setuparg_list_fields = [
|
||||
'Classifier',
|
||||
'Requires',
|
||||
'Provides',
|
||||
'Obsoletes',
|
||||
'Platform',
|
||||
'Supported-Platform',
|
||||
]
|
||||
setuparg_multi_line_values = ['Description']
|
||||
replacements = [
|
||||
('License', r' +$', ''),
|
||||
('License', r'^ +', ''),
|
||||
('License', r' ', '-'),
|
||||
('License', r'^GNU-', ''),
|
||||
('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
|
||||
('License', r'^UNKNOWN$', ''),
|
||||
|
||||
# Remove currently unhandled version numbers from these variables
|
||||
('Requires', r' *\([^)]*\)', ''),
|
||||
('Provides', r' *\([^)]*\)', ''),
|
||||
('Obsoletes', r' *\([^)]*\)', ''),
|
||||
('Install-requires', r'^([^><= ]+).*', r'\1'),
|
||||
('Extras-require', r'^([^><= ]+).*', r'\1'),
|
||||
('Tests-require', r'^([^><= ]+).*', r'\1'),
|
||||
|
||||
# Remove unhandled dependency on particular features (e.g. foo[PDF])
|
||||
('Install-requires', r'\[[^\]]+\]$', ''),
|
||||
]
|
||||
|
||||
classifier_license_map = {
|
||||
'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
|
||||
|
|
@ -166,122 +111,34 @@ class PythonRecipeHandler(RecipeHandler):
|
|||
def __init__(self):
|
||||
pass
|
||||
|
||||
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
||||
if 'buildsystem' in handled:
|
||||
return False
|
||||
def handle_classifier_license(self, classifiers, existing_licenses=""):
|
||||
|
||||
# Check for non-zero size setup.py files
|
||||
setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
|
||||
for fn in setupfiles:
|
||||
if os.path.getsize(fn):
|
||||
break
|
||||
else:
|
||||
return False
|
||||
licenses = []
|
||||
for classifier in classifiers:
|
||||
if classifier in self.classifier_license_map:
|
||||
license = self.classifier_license_map[classifier]
|
||||
if license == 'Apache' and 'Apache-2.0' in existing_licenses:
|
||||
license = 'Apache-2.0'
|
||||
elif license == 'GPL':
|
||||
if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
|
||||
license = 'GPL-2.0'
|
||||
elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
|
||||
license = 'GPL-3.0'
|
||||
elif license == 'LGPL':
|
||||
if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
|
||||
license = 'LGPL-2.1'
|
||||
elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
|
||||
license = 'LGPL-2.0'
|
||||
elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
|
||||
license = 'LGPL-3.0'
|
||||
licenses.append(license)
|
||||
|
||||
# setup.py is always parsed to get at certain required information, such as
|
||||
# distutils vs setuptools
|
||||
#
|
||||
# If egg info is available, we use it for both its PKG-INFO metadata
|
||||
# and for its requires.txt for install_requires.
|
||||
# If PKG-INFO is available but no egg info is, we use that for metadata in preference to
|
||||
# the parsed setup.py, but use the install_requires info from the
|
||||
# parsed setup.py.
|
||||
if licenses:
|
||||
return ' & '.join(licenses)
|
||||
|
||||
setupscript = os.path.join(srctree, 'setup.py')
|
||||
try:
|
||||
setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
|
||||
except Exception:
|
||||
logger.exception("Failed to parse setup.py")
|
||||
setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
|
||||
return None
|
||||
|
||||
egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
|
||||
if egginfo:
|
||||
info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
|
||||
requires_txt = os.path.join(egginfo[0], 'requires.txt')
|
||||
if os.path.exists(requires_txt):
|
||||
with codecs.open(requires_txt) as f:
|
||||
inst_req = []
|
||||
extras_req = collections.defaultdict(list)
|
||||
current_feature = None
|
||||
for line in f.readlines():
|
||||
line = line.rstrip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
if line.startswith('['):
|
||||
# PACKAGECONFIG must not contain expressions or whitespace
|
||||
line = line.replace(" ", "")
|
||||
line = line.replace(':', "")
|
||||
line = line.replace('.', "-dot-")
|
||||
line = line.replace('"', "")
|
||||
line = line.replace('<', "-smaller-")
|
||||
line = line.replace('>', "-bigger-")
|
||||
line = line.replace('_', "-")
|
||||
line = line.replace('(', "")
|
||||
line = line.replace(')', "")
|
||||
line = line.replace('!', "-not-")
|
||||
line = line.replace('=', "-equals-")
|
||||
current_feature = line[1:-1]
|
||||
elif current_feature:
|
||||
extras_req[current_feature].append(line)
|
||||
else:
|
||||
inst_req.append(line)
|
||||
info['Install-requires'] = inst_req
|
||||
info['Extras-require'] = extras_req
|
||||
elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
|
||||
info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
|
||||
|
||||
if setup_info:
|
||||
if 'Install-requires' in setup_info:
|
||||
info['Install-requires'] = setup_info['Install-requires']
|
||||
if 'Extras-require' in setup_info:
|
||||
info['Extras-require'] = setup_info['Extras-require']
|
||||
else:
|
||||
if setup_info:
|
||||
info = setup_info
|
||||
else:
|
||||
info = self.get_setup_args_info(setupscript)
|
||||
|
||||
# Grab the license value before applying replacements
|
||||
license_str = info.get('License', '').strip()
|
||||
|
||||
self.apply_info_replacements(info)
|
||||
|
||||
if uses_setuptools:
|
||||
classes.append('setuptools3')
|
||||
else:
|
||||
classes.append('distutils3')
|
||||
|
||||
if license_str:
|
||||
for i, line in enumerate(lines_before):
|
||||
if line.startswith('##LICENSE_PLACEHOLDER##'):
|
||||
lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
|
||||
break
|
||||
|
||||
if 'Classifier' in info:
|
||||
existing_licenses = info.get('License', '')
|
||||
licenses = []
|
||||
for classifier in info['Classifier']:
|
||||
if classifier in self.classifier_license_map:
|
||||
license = self.classifier_license_map[classifier]
|
||||
if license == 'Apache' and 'Apache-2.0' in existing_licenses:
|
||||
license = 'Apache-2.0'
|
||||
elif license == 'GPL':
|
||||
if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
|
||||
license = 'GPL-2.0'
|
||||
elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
|
||||
license = 'GPL-3.0'
|
||||
elif license == 'LGPL':
|
||||
if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
|
||||
license = 'LGPL-2.1'
|
||||
elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
|
||||
license = 'LGPL-2.0'
|
||||
elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
|
||||
license = 'LGPL-3.0'
|
||||
licenses.append(license)
|
||||
|
||||
if licenses:
|
||||
info['License'] = ' & '.join(licenses)
|
||||
def map_info_to_bbvar(self, info, extravalues):
|
||||
|
||||
# Map PKG-INFO & setup.py fields to bitbake variables
|
||||
for field, values in info.items():
|
||||
|
|
@ -305,147 +162,10 @@ class PythonRecipeHandler(RecipeHandler):
|
|||
if bbvar not in extravalues and value:
|
||||
extravalues[bbvar] = value
|
||||
|
||||
mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
|
||||
|
||||
extras_req = set()
|
||||
if 'Extras-require' in info:
|
||||
extras_req = info['Extras-require']
|
||||
if extras_req:
|
||||
lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
|
||||
lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
|
||||
lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
|
||||
lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
|
||||
lines_after.append('#')
|
||||
lines_after.append('# Uncomment this line to enable all the optional features.')
|
||||
lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
|
||||
for feature, feature_reqs in extras_req.items():
|
||||
unmapped_deps.difference_update(feature_reqs)
|
||||
|
||||
feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
|
||||
lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
|
||||
|
||||
inst_reqs = set()
|
||||
if 'Install-requires' in info:
|
||||
if extras_req:
|
||||
lines_after.append('')
|
||||
inst_reqs = info['Install-requires']
|
||||
if inst_reqs:
|
||||
unmapped_deps.difference_update(inst_reqs)
|
||||
|
||||
inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
|
||||
lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
|
||||
lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
|
||||
lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
|
||||
|
||||
if mapped_deps:
|
||||
name = info.get('Name')
|
||||
if name and name[0] in mapped_deps:
|
||||
# Attempt to avoid self-reference
|
||||
mapped_deps.remove(name[0])
|
||||
mapped_deps -= set(self.excluded_pkgdeps)
|
||||
if inst_reqs or extras_req:
|
||||
lines_after.append('')
|
||||
lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
|
||||
lines_after.append('# python sources, and might not be 100% accurate.')
|
||||
lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
|
||||
|
||||
unmapped_deps -= set(extensions)
|
||||
unmapped_deps -= set(self.assume_provided)
|
||||
if unmapped_deps:
|
||||
if mapped_deps:
|
||||
lines_after.append('')
|
||||
lines_after.append('# WARNING: We were unable to map the following python package/module')
|
||||
lines_after.append('# dependencies to the bitbake packages which include them:')
|
||||
lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps))
|
||||
|
||||
handled.append('buildsystem')
|
||||
|
||||
def get_pkginfo(self, pkginfo_fn):
|
||||
msg = email.message_from_file(open(pkginfo_fn, 'r'))
|
||||
msginfo = {}
|
||||
for field in msg.keys():
|
||||
values = msg.get_all(field)
|
||||
if len(values) == 1:
|
||||
msginfo[field] = values[0]
|
||||
else:
|
||||
msginfo[field] = values
|
||||
return msginfo
|
||||
|
||||
def parse_setup_py(self, setupscript='./setup.py'):
|
||||
with codecs.open(setupscript) as f:
|
||||
info, imported_modules, non_literals, extensions = gather_setup_info(f)
|
||||
|
||||
def _map(key):
|
||||
key = key.replace('_', '-')
|
||||
key = key[0].upper() + key[1:]
|
||||
if key in self.setup_parse_map:
|
||||
key = self.setup_parse_map[key]
|
||||
return key
|
||||
|
||||
# Naive mapping of setup() arguments to PKG-INFO field names
|
||||
for d in [info, non_literals]:
|
||||
for key, value in list(d.items()):
|
||||
if key is None:
|
||||
continue
|
||||
new_key = _map(key)
|
||||
if new_key != key:
|
||||
del d[key]
|
||||
d[new_key] = value
|
||||
|
||||
return info, 'setuptools' in imported_modules, non_literals, extensions
|
||||
|
||||
def get_setup_args_info(self, setupscript='./setup.py'):
|
||||
cmd = ['python3', setupscript]
|
||||
info = {}
|
||||
keys = set(self.bbvar_map.keys())
|
||||
keys |= set(self.setuparg_list_fields)
|
||||
keys |= set(self.setuparg_multi_line_values)
|
||||
grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
|
||||
for index, keys in grouped_keys:
|
||||
if index == (True, False):
|
||||
# Splitlines output for each arg as a list value
|
||||
for key in keys:
|
||||
arg = self.setuparg_map.get(key, key.lower())
|
||||
try:
|
||||
arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
else:
|
||||
info[key] = [l.rstrip() for l in arg_info.splitlines()]
|
||||
elif index == (False, True):
|
||||
# Entire output for each arg
|
||||
for key in keys:
|
||||
arg = self.setuparg_map.get(key, key.lower())
|
||||
try:
|
||||
arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
else:
|
||||
info[key] = arg_info
|
||||
else:
|
||||
info.update(self.get_setup_byline(list(keys), setupscript))
|
||||
return info
|
||||
|
||||
def get_setup_byline(self, fields, setupscript='./setup.py'):
|
||||
info = {}
|
||||
|
||||
cmd = ['python3', setupscript]
|
||||
cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
|
||||
try:
|
||||
info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
else:
|
||||
if len(fields) != len(info_lines):
|
||||
logger.error('Mismatch between setup.py output lines and number of fields')
|
||||
sys.exit(1)
|
||||
|
||||
for lineno, line in enumerate(info_lines):
|
||||
line = line.rstrip()
|
||||
info[fields[lineno]] = line
|
||||
return info
|
||||
|
||||
def apply_info_replacements(self, info):
|
||||
if not self.replacements:
|
||||
return
|
||||
|
||||
for variable, search, replace in self.replacements:
|
||||
if variable not in info:
|
||||
continue
|
||||
|
|
@ -487,58 +207,6 @@ class PythonRecipeHandler(RecipeHandler):
|
|||
if value != new_list:
|
||||
info[variable] = new_list
|
||||
|
||||
def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
|
||||
if 'Package-dir' in setup_info:
|
||||
package_dir = setup_info['Package-dir']
|
||||
else:
|
||||
package_dir = {}
|
||||
|
||||
dist = setuptools.Distribution()
|
||||
|
||||
class PackageDir(setuptools.command.build_py.build_py):
|
||||
def __init__(self, package_dir):
|
||||
self.package_dir = package_dir
|
||||
self.dist = dist
|
||||
super().__init__(self.dist)
|
||||
|
||||
pd = PackageDir(package_dir)
|
||||
to_scan = []
|
||||
if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
|
||||
if 'Py-modules' in setup_info:
|
||||
for module in setup_info['Py-modules']:
|
||||
try:
|
||||
package, module = module.rsplit('.', 1)
|
||||
except ValueError:
|
||||
package, module = '.', module
|
||||
module_path = os.path.join(pd.get_package_dir(package), module + '.py')
|
||||
to_scan.append(module_path)
|
||||
|
||||
if 'Packages' in setup_info:
|
||||
for package in setup_info['Packages']:
|
||||
to_scan.append(pd.get_package_dir(package))
|
||||
|
||||
if 'Scripts' in setup_info:
|
||||
to_scan.extend(setup_info['Scripts'])
|
||||
else:
|
||||
logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
|
||||
|
||||
if not to_scan:
|
||||
to_scan = ['.']
|
||||
|
||||
logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
|
||||
|
||||
provided_packages = self.parse_pkgdata_for_python_packages()
|
||||
scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
|
||||
mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
|
||||
for dep in scanned_deps:
|
||||
mapped = provided_packages.get(dep)
|
||||
if mapped:
|
||||
logger.debug('Mapped %s to %s' % (dep, mapped))
|
||||
mapped_deps.add(mapped)
|
||||
else:
|
||||
logger.debug('Could not map %s' % dep)
|
||||
unmapped_deps.add(dep)
|
||||
return mapped_deps, unmapped_deps
|
||||
|
||||
def scan_python_dependencies(self, paths):
|
||||
deps = set()
|
||||
|
|
@ -633,6 +301,360 @@ class PythonRecipeHandler(RecipeHandler):
|
|||
logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
|
||||
raise
|
||||
|
||||
class PythonSetupPyRecipeHandler(PythonRecipeHandler):
|
||||
bbvar_map = {
|
||||
'Name': 'PN',
|
||||
'Version': 'PV',
|
||||
'Home-page': 'HOMEPAGE',
|
||||
'Summary': 'SUMMARY',
|
||||
'Description': 'DESCRIPTION',
|
||||
'License': 'LICENSE',
|
||||
'Requires': 'RDEPENDS:${PN}',
|
||||
'Provides': 'RPROVIDES:${PN}',
|
||||
'Obsoletes': 'RREPLACES:${PN}',
|
||||
}
|
||||
# PN/PV are already set by recipetool core & desc can be extremely long
|
||||
excluded_fields = [
|
||||
'Description',
|
||||
]
|
||||
setup_parse_map = {
|
||||
'Url': 'Home-page',
|
||||
'Classifiers': 'Classifier',
|
||||
'Description': 'Summary',
|
||||
}
|
||||
setuparg_map = {
|
||||
'Home-page': 'url',
|
||||
'Classifier': 'classifiers',
|
||||
'Summary': 'description',
|
||||
'Description': 'long-description',
|
||||
}
|
||||
# Values which are lists, used by the setup.py argument based metadata
|
||||
# extraction method, to determine how to process the setup.py output.
|
||||
setuparg_list_fields = [
|
||||
'Classifier',
|
||||
'Requires',
|
||||
'Provides',
|
||||
'Obsoletes',
|
||||
'Platform',
|
||||
'Supported-Platform',
|
||||
]
|
||||
setuparg_multi_line_values = ['Description']
|
||||
|
||||
replacements = [
|
||||
('License', r' +$', ''),
|
||||
('License', r'^ +', ''),
|
||||
('License', r' ', '-'),
|
||||
('License', r'^GNU-', ''),
|
||||
('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
|
||||
('License', r'^UNKNOWN$', ''),
|
||||
|
||||
# Remove currently unhandled version numbers from these variables
|
||||
('Requires', r' *\([^)]*\)', ''),
|
||||
('Provides', r' *\([^)]*\)', ''),
|
||||
('Obsoletes', r' *\([^)]*\)', ''),
|
||||
('Install-requires', r'^([^><= ]+).*', r'\1'),
|
||||
('Extras-require', r'^([^><= ]+).*', r'\1'),
|
||||
('Tests-require', r'^([^><= ]+).*', r'\1'),
|
||||
|
||||
# Remove unhandled dependency on particular features (e.g. foo[PDF])
|
||||
('Install-requires', r'\[[^\]]+\]$', ''),
|
||||
]
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def parse_setup_py(self, setupscript='./setup.py'):
|
||||
with codecs.open(setupscript) as f:
|
||||
info, imported_modules, non_literals, extensions = gather_setup_info(f)
|
||||
|
||||
def _map(key):
|
||||
key = key.replace('_', '-')
|
||||
key = key[0].upper() + key[1:]
|
||||
if key in self.setup_parse_map:
|
||||
key = self.setup_parse_map[key]
|
||||
return key
|
||||
|
||||
# Naive mapping of setup() arguments to PKG-INFO field names
|
||||
for d in [info, non_literals]:
|
||||
for key, value in list(d.items()):
|
||||
if key is None:
|
||||
continue
|
||||
new_key = _map(key)
|
||||
if new_key != key:
|
||||
del d[key]
|
||||
d[new_key] = value
|
||||
|
||||
return info, 'setuptools' in imported_modules, non_literals, extensions
|
||||
|
||||
def get_setup_args_info(self, setupscript='./setup.py'):
|
||||
cmd = ['python3', setupscript]
|
||||
info = {}
|
||||
keys = set(self.bbvar_map.keys())
|
||||
keys |= set(self.setuparg_list_fields)
|
||||
keys |= set(self.setuparg_multi_line_values)
|
||||
grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
|
||||
for index, keys in grouped_keys:
|
||||
if index == (True, False):
|
||||
# Splitlines output for each arg as a list value
|
||||
for key in keys:
|
||||
arg = self.setuparg_map.get(key, key.lower())
|
||||
try:
|
||||
arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
else:
|
||||
info[key] = [l.rstrip() for l in arg_info.splitlines()]
|
||||
elif index == (False, True):
|
||||
# Entire output for each arg
|
||||
for key in keys:
|
||||
arg = self.setuparg_map.get(key, key.lower())
|
||||
try:
|
||||
arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
else:
|
||||
info[key] = arg_info
|
||||
else:
|
||||
info.update(self.get_setup_byline(list(keys), setupscript))
|
||||
return info
|
||||
|
||||
def get_setup_byline(self, fields, setupscript='./setup.py'):
|
||||
info = {}
|
||||
|
||||
cmd = ['python3', setupscript]
|
||||
cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
|
||||
try:
|
||||
info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
pass
|
||||
else:
|
||||
if len(fields) != len(info_lines):
|
||||
logger.error('Mismatch between setup.py output lines and number of fields')
|
||||
sys.exit(1)
|
||||
|
||||
for lineno, line in enumerate(info_lines):
|
||||
line = line.rstrip()
|
||||
info[fields[lineno]] = line
|
||||
return info
|
||||
|
||||
def get_pkginfo(self, pkginfo_fn):
|
||||
msg = email.message_from_file(open(pkginfo_fn, 'r'))
|
||||
msginfo = {}
|
||||
for field in msg.keys():
|
||||
values = msg.get_all(field)
|
||||
if len(values) == 1:
|
||||
msginfo[field] = values[0]
|
||||
else:
|
||||
msginfo[field] = values
|
||||
return msginfo
|
||||
|
||||
def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
|
||||
if 'Package-dir' in setup_info:
|
||||
package_dir = setup_info['Package-dir']
|
||||
else:
|
||||
package_dir = {}
|
||||
|
||||
dist = setuptools.Distribution()
|
||||
|
||||
class PackageDir(setuptools.command.build_py.build_py):
|
||||
def __init__(self, package_dir):
|
||||
self.package_dir = package_dir
|
||||
self.dist = dist
|
||||
super().__init__(self.dist)
|
||||
|
||||
pd = PackageDir(package_dir)
|
||||
to_scan = []
|
||||
if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
|
||||
if 'Py-modules' in setup_info:
|
||||
for module in setup_info['Py-modules']:
|
||||
try:
|
||||
package, module = module.rsplit('.', 1)
|
||||
except ValueError:
|
||||
package, module = '.', module
|
||||
module_path = os.path.join(pd.get_package_dir(package), module + '.py')
|
||||
to_scan.append(module_path)
|
||||
|
||||
if 'Packages' in setup_info:
|
||||
for package in setup_info['Packages']:
|
||||
to_scan.append(pd.get_package_dir(package))
|
||||
|
||||
if 'Scripts' in setup_info:
|
||||
to_scan.extend(setup_info['Scripts'])
|
||||
else:
|
||||
logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
|
||||
|
||||
if not to_scan:
|
||||
to_scan = ['.']
|
||||
|
||||
logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
|
||||
|
||||
provided_packages = self.parse_pkgdata_for_python_packages()
|
||||
scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
|
||||
mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
|
||||
for dep in scanned_deps:
|
||||
mapped = provided_packages.get(dep)
|
||||
if mapped:
|
||||
logger.debug('Mapped %s to %s' % (dep, mapped))
|
||||
mapped_deps.add(mapped)
|
||||
else:
|
||||
logger.debug('Could not map %s' % dep)
|
||||
unmapped_deps.add(dep)
|
||||
return mapped_deps, unmapped_deps
|
||||
|
||||
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
||||
|
||||
if 'buildsystem' in handled:
|
||||
return False
|
||||
|
||||
# Check for non-zero size setup.py files
|
||||
setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
|
||||
for fn in setupfiles:
|
||||
if os.path.getsize(fn):
|
||||
break
|
||||
else:
|
||||
return False
|
||||
|
||||
# setup.py is always parsed to get at certain required information, such as
|
||||
# distutils vs setuptools
|
||||
#
|
||||
# If egg info is available, we use it for both its PKG-INFO metadata
|
||||
# and for its requires.txt for install_requires.
|
||||
# If PKG-INFO is available but no egg info is, we use that for metadata in preference to
|
||||
# the parsed setup.py, but use the install_requires info from the
|
||||
# parsed setup.py.
|
||||
|
||||
setupscript = os.path.join(srctree, 'setup.py')
|
||||
try:
|
||||
setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
|
||||
except Exception:
|
||||
logger.exception("Failed to parse setup.py")
|
||||
setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
|
||||
|
||||
egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
|
||||
if egginfo:
|
||||
info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
|
||||
requires_txt = os.path.join(egginfo[0], 'requires.txt')
|
||||
if os.path.exists(requires_txt):
|
||||
with codecs.open(requires_txt) as f:
|
||||
inst_req = []
|
||||
extras_req = collections.defaultdict(list)
|
||||
current_feature = None
|
||||
for line in f.readlines():
|
||||
line = line.rstrip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
if line.startswith('['):
|
||||
# PACKAGECONFIG must not contain expressions or whitespace
|
||||
line = line.replace(" ", "")
|
||||
line = line.replace(':', "")
|
||||
line = line.replace('.', "-dot-")
|
||||
line = line.replace('"', "")
|
||||
line = line.replace('<', "-smaller-")
|
||||
line = line.replace('>', "-bigger-")
|
||||
line = line.replace('_', "-")
|
||||
line = line.replace('(', "")
|
||||
line = line.replace(')', "")
|
||||
line = line.replace('!', "-not-")
|
||||
line = line.replace('=', "-equals-")
|
||||
current_feature = line[1:-1]
|
||||
elif current_feature:
|
||||
extras_req[current_feature].append(line)
|
||||
else:
|
||||
inst_req.append(line)
|
||||
info['Install-requires'] = inst_req
|
||||
info['Extras-require'] = extras_req
|
||||
elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
|
||||
info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
|
||||
|
||||
if setup_info:
|
||||
if 'Install-requires' in setup_info:
|
||||
info['Install-requires'] = setup_info['Install-requires']
|
||||
if 'Extras-require' in setup_info:
|
||||
info['Extras-require'] = setup_info['Extras-require']
|
||||
else:
|
||||
if setup_info:
|
||||
info = setup_info
|
||||
else:
|
||||
info = self.get_setup_args_info(setupscript)
|
||||
|
||||
# Grab the license value before applying replacements
|
||||
license_str = info.get('License', '').strip()
|
||||
|
||||
self.apply_info_replacements(info)
|
||||
|
||||
if uses_setuptools:
|
||||
classes.append('setuptools3')
|
||||
else:
|
||||
classes.append('distutils3')
|
||||
|
||||
if license_str:
|
||||
for i, line in enumerate(lines_before):
|
||||
if line.startswith('##LICENSE_PLACEHOLDER##'):
|
||||
lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
|
||||
break
|
||||
|
||||
if 'Classifier' in info:
|
||||
license = self.handle_classifier_license(info['Classifier'], info.get('License', ''))
|
||||
if license:
|
||||
info['License'] = license
|
||||
|
||||
self.map_info_to_bbvar(info, extravalues)
|
||||
|
||||
mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
|
||||
|
||||
extras_req = set()
|
||||
if 'Extras-require' in info:
|
||||
extras_req = info['Extras-require']
|
||||
if extras_req:
|
||||
lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
|
||||
lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
|
||||
lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
|
||||
lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
|
||||
lines_after.append('#')
|
||||
lines_after.append('# Uncomment this line to enable all the optional features.')
|
||||
lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
|
||||
for feature, feature_reqs in extras_req.items():
|
||||
unmapped_deps.difference_update(feature_reqs)
|
||||
|
||||
feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
|
||||
lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
|
||||
|
||||
inst_reqs = set()
|
||||
if 'Install-requires' in info:
|
||||
if extras_req:
|
||||
lines_after.append('')
|
||||
inst_reqs = info['Install-requires']
|
||||
if inst_reqs:
|
||||
unmapped_deps.difference_update(inst_reqs)
|
||||
|
||||
inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
|
||||
lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
|
||||
lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
|
||||
lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
|
||||
|
||||
if mapped_deps:
|
||||
name = info.get('Name')
|
||||
if name and name[0] in mapped_deps:
|
||||
# Attempt to avoid self-reference
|
||||
mapped_deps.remove(name[0])
|
||||
mapped_deps -= set(self.excluded_pkgdeps)
|
||||
if inst_reqs or extras_req:
|
||||
lines_after.append('')
|
||||
lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
|
||||
lines_after.append('# python sources, and might not be 100% accurate.')
|
||||
lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
|
||||
|
||||
unmapped_deps -= set(extensions)
|
||||
unmapped_deps -= set(self.assume_provided)
|
||||
if unmapped_deps:
|
||||
if mapped_deps:
|
||||
lines_after.append('')
|
||||
lines_after.append('# WARNING: We were unable to map the following python package/module')
|
||||
lines_after.append('# dependencies to the bitbake packages which include them:')
|
||||
lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps))
|
||||
|
||||
handled.append('buildsystem')
|
||||
|
||||
def gather_setup_info(fileobj):
|
||||
parsed = ast.parse(fileobj.read(), fileobj.name)
|
||||
|
|
@ -748,4 +770,4 @@ def has_non_literals(value):
|
|||
|
||||
def register_recipe_handlers(handlers):
|
||||
# We need to make sure this is ahead of the makefile fallback handler
|
||||
handlers.append((PythonRecipeHandler(), 70))
|
||||
handlers.append((PythonSetupPyRecipeHandler(), 70))
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user