mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
bb8f141d0c
The change over to recipe specific sysroots means that we can no longer get a known location simply from configuration for the npm binary - we need to get the recipe sysroot for nodejs-native, look there for npm if we need to check it's present, and add that to PATH when calling out to npm. Unfortunately this means anywhere we need to get that path we have to have parsed all recipes, otherwise we have no reliable way of resolving nodejs-native. Thus we have to change recipetool create to always parse all recipes (the structure of the code does not allow us to do this conditionally). In the worst case, if npm hasn't already been added to its own sysroot and we are fetching from a source repository rather than an npm registry, this gets a bit ugly because we end up parsing recipes three times: 1) recipetool startup, which then fetches the code and determines it's a node.js module, finds that npm isn't available and then exits with a specific error to tell devtool it needs to build npm 2) when we invoke bitbake -c addto_recipe_sysroot nodejs-native 3) when we re-invoke recipetool This code is badly in need of refactoring, but now is unfortunately not the time to do that, so we're going to have to live with this ugliness for now. Fixes [YOCTO #10992]. (From OE-Core rev: acfdbd796c99882b8586023c8c6b848716105c8d) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
345 lines
15 KiB
Python
345 lines
15 KiB
Python
# Recipe creation tool - node.js NPM module support plugin
|
|
#
|
|
# Copyright (C) 2016 Intel Corporation
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import os
|
|
import logging
|
|
import subprocess
|
|
import tempfile
|
|
import shutil
|
|
import json
|
|
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm
|
|
|
|
logger = logging.getLogger('recipetool')
|
|
|
|
|
|
tinfoil = None
|
|
|
|
def tinfoil_init(instance):
|
|
global tinfoil
|
|
tinfoil = instance
|
|
|
|
|
|
class NpmRecipeHandler(RecipeHandler):
|
|
lockdownpath = None
|
|
|
|
def _handle_license(self, data):
|
|
'''
|
|
Handle the license value from an npm package.json file
|
|
'''
|
|
license = None
|
|
if 'license' in data:
|
|
license = data['license']
|
|
if isinstance(license, dict):
|
|
license = license.get('type', None)
|
|
if license:
|
|
if 'OR' in license:
|
|
license = license.replace('OR', '|')
|
|
license = license.replace('AND', '&')
|
|
license = license.replace(' ', '_')
|
|
if not license[0] == '(':
|
|
license = '(' + license + ')'
|
|
print('LICENSE: {}'.format(license))
|
|
else:
|
|
license = license.replace('AND', '&')
|
|
if license[0] == '(':
|
|
license = license[1:]
|
|
if license[-1] == ')':
|
|
license = license[:-1]
|
|
license = license.replace('MIT/X11', 'MIT')
|
|
license = license.replace('Public Domain', 'PD')
|
|
license = license.replace('SEE LICENSE IN EULA',
|
|
'SEE-LICENSE-IN-EULA')
|
|
return license
|
|
|
|
def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
|
|
try:
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
|
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
except bb.process.ExecutionError as e:
|
|
logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
|
|
return
|
|
|
|
tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
|
|
shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
|
|
extravalues.setdefault('extrafiles', {})
|
|
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
|
|
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
|
|
|
|
def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
|
if not NpmRecipeHandler.lockdownpath:
|
|
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
|
|
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
|
|
cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
|
|
if not os.path.exists(relockbin):
|
|
logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
|
|
return
|
|
try:
|
|
bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
except bb.process.ExecutionError as e:
|
|
logger.warn('lockdown-relock failed:\n%s' % e.stdout)
|
|
return
|
|
|
|
tmpfile = os.path.join(localfilesdir, 'lockdown.json')
|
|
shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
|
|
extravalues.setdefault('extrafiles', {})
|
|
extravalues['extrafiles']['lockdown.json'] = tmpfile
|
|
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
|
|
|
|
def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
|
|
import scriptutils
|
|
# If this isn't a single module we need to get the dependencies
|
|
# and add them to SRC_URI
|
|
def varfunc(varname, origvalue, op, newlines):
|
|
if varname == 'SRC_URI':
|
|
if not origvalue.startswith('npm://'):
|
|
src_uri = origvalue.split()
|
|
changed = False
|
|
deplist = {}
|
|
for dep, depver in optdeps.items():
|
|
depdata = self.get_npm_data(dep, depver, d)
|
|
if self.check_npm_optional_dependency(depdata):
|
|
deplist[dep] = depdata
|
|
for dep, depver in devdeps.items():
|
|
depdata = self.get_npm_data(dep, depver, d)
|
|
if self.check_npm_optional_dependency(depdata):
|
|
deplist[dep] = depdata
|
|
for dep, depver in deps.items():
|
|
depdata = self.get_npm_data(dep, depver, d)
|
|
deplist[dep] = depdata
|
|
|
|
for dep, depdata in deplist.items():
|
|
version = depdata.get('version', None)
|
|
if version:
|
|
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
|
|
scriptutils.fetch_uri(d, url, srctree)
|
|
src_uri.append(url)
|
|
changed = True
|
|
if changed:
|
|
return src_uri, None, -1, True
|
|
return origvalue, None, 0, True
|
|
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
|
|
if updated:
|
|
del lines_before[:]
|
|
for line in newlines:
|
|
# Hack to avoid newlines that edit_metadata inserts
|
|
if line.endswith('\n'):
|
|
line = line[:-1]
|
|
lines_before.append(line)
|
|
return updated
|
|
|
|
def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d):
|
|
for item in handled:
|
|
if isinstance(item, tuple):
|
|
if item[0] == 'license':
|
|
del item
|
|
break
|
|
|
|
calledvars = []
|
|
def varfunc(varname, origvalue, op, newlines):
|
|
if varname in ['LICENSE', 'LIC_FILES_CHKSUM']:
|
|
for i, e in enumerate(reversed(newlines)):
|
|
if not e.startswith('#'):
|
|
stop = i
|
|
while stop > 0:
|
|
newlines.pop()
|
|
stop -= 1
|
|
break
|
|
calledvars.append(varname)
|
|
if len(calledvars) > 1:
|
|
# The second time around, put the new license text in
|
|
insertpos = len(newlines)
|
|
handle_license_vars(srctree, newlines, handled, extravalues, d)
|
|
return None, None, 0, True
|
|
return origvalue, None, 0, True
|
|
updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
|
|
if updated:
|
|
del lines_before[:]
|
|
lines_before.extend(newlines)
|
|
else:
|
|
raise Exception('Did not find license variables')
|
|
|
|
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
|
import bb.utils
|
|
import oe
|
|
from collections import OrderedDict
|
|
|
|
if 'buildsystem' in handled:
|
|
return False
|
|
|
|
def read_package_json(fn):
|
|
with open(fn, 'r', errors='surrogateescape') as f:
|
|
return json.loads(f.read())
|
|
|
|
files = RecipeHandler.checkfiles(srctree, ['package.json'])
|
|
if files:
|
|
d = bb.data.createCopy(tinfoil.config_data)
|
|
npm_bindir = check_npm(tinfoil)
|
|
d.prependVar('PATH', '%s:' % npm_bindir)
|
|
|
|
data = read_package_json(files[0])
|
|
if 'name' in data and 'version' in data:
|
|
extravalues['PN'] = data['name']
|
|
extravalues['PV'] = data['version']
|
|
classes.append('npm')
|
|
handled.append('buildsystem')
|
|
if 'description' in data:
|
|
extravalues['SUMMARY'] = data['description']
|
|
if 'homepage' in data:
|
|
extravalues['HOMEPAGE'] = data['homepage']
|
|
|
|
fetchdev = extravalues['fetchdev'] or None
|
|
deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
|
|
updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
|
|
if updated:
|
|
# We need to redo the license stuff
|
|
self._replace_license_vars(srctree, lines_before, handled, extravalues, d)
|
|
|
|
# Shrinkwrap
|
|
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
|
|
self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
|
|
|
|
# Lockdown
|
|
self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
|
|
|
|
# Split each npm module out to is own package
|
|
npmpackages = oe.package.npm_split_package_dirs(srctree)
|
|
for item in handled:
|
|
if isinstance(item, tuple):
|
|
if item[0] == 'license':
|
|
licvalues = item[1]
|
|
break
|
|
if licvalues:
|
|
# Augment the license list with information we have in the packages
|
|
licenses = {}
|
|
license = self._handle_license(data)
|
|
if license:
|
|
licenses['${PN}'] = license
|
|
for pkgname, pkgitem in npmpackages.items():
|
|
_, pdata = pkgitem
|
|
license = self._handle_license(pdata)
|
|
if license:
|
|
licenses[pkgname] = license
|
|
# Now write out the package-specific license values
|
|
# We need to strip out the json data dicts for this since split_pkg_licenses
|
|
# isn't expecting it
|
|
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
|
|
packages['${PN}'] = ''
|
|
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
|
|
all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
|
|
if '&' in all_licenses:
|
|
all_licenses.remove('&')
|
|
# Go back and update the LICENSE value since we have a bit more
|
|
# information than when that was written out (and we know all apply
|
|
# vs. there being a choice, so we can join them with &)
|
|
for i, line in enumerate(lines_before):
|
|
if line.startswith('LICENSE = '):
|
|
lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses)
|
|
break
|
|
|
|
# Need to move S setting after inherit npm
|
|
for i, line in enumerate(lines_before):
|
|
if line.startswith('S ='):
|
|
lines_before.pop(i)
|
|
lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
|
|
lines_after.insert(1, line)
|
|
break
|
|
|
|
return True
|
|
|
|
return False
|
|
|
|
# FIXME this is duplicated from lib/bb/fetch2/npm.py
|
|
def _parse_view(self, output):
|
|
'''
|
|
Parse the output of npm view --json; the last JSON result
|
|
is assumed to be the one that we're interested in.
|
|
'''
|
|
pdata = None
|
|
outdeps = {}
|
|
datalines = []
|
|
bracelevel = 0
|
|
for line in output.splitlines():
|
|
if bracelevel:
|
|
datalines.append(line)
|
|
elif '{' in line:
|
|
datalines = []
|
|
datalines.append(line)
|
|
bracelevel = bracelevel + line.count('{') - line.count('}')
|
|
if datalines:
|
|
pdata = json.loads('\n'.join(datalines))
|
|
return pdata
|
|
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
|
# (split out from _getdependencies())
|
|
def get_npm_data(self, pkg, version, d):
|
|
import bb.fetch2
|
|
pkgfullname = pkg
|
|
if version != '*' and not '/' in version:
|
|
pkgfullname += "@'%s'" % version
|
|
logger.debug(2, "Calling getdeps on %s" % pkg)
|
|
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
|
fetchcmd = "npm view %s --json" % pkgfullname
|
|
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
|
data = self._parse_view(output)
|
|
return data
|
|
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
|
# (split out from _getdependencies())
|
|
def get_npm_package_dependencies(self, pdata, fetchdev):
|
|
dependencies = pdata.get('dependencies', {})
|
|
optionalDependencies = pdata.get('optionalDependencies', {})
|
|
dependencies.update(optionalDependencies)
|
|
if fetchdev:
|
|
devDependencies = pdata.get('devDependencies', {})
|
|
dependencies.update(devDependencies)
|
|
else:
|
|
devDependencies = {}
|
|
depsfound = {}
|
|
optdepsfound = {}
|
|
devdepsfound = {}
|
|
for dep in dependencies:
|
|
if dep in optionalDependencies:
|
|
optdepsfound[dep] = dependencies[dep]
|
|
elif dep in devDependencies:
|
|
devdepsfound[dep] = dependencies[dep]
|
|
else:
|
|
depsfound[dep] = dependencies[dep]
|
|
return depsfound, optdepsfound, devdepsfound
|
|
|
|
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
|
# (split out from _getdependencies())
|
|
def check_npm_optional_dependency(self, pdata):
|
|
pkg_os = pdata.get('os', None)
|
|
if pkg_os:
|
|
if not isinstance(pkg_os, list):
|
|
pkg_os = [pkg_os]
|
|
blacklist = False
|
|
for item in pkg_os:
|
|
if item.startswith('!'):
|
|
blacklist = True
|
|
break
|
|
if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
|
|
logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
|
|
return False
|
|
return True
|
|
|
|
|
|
def register_recipe_handlers(handlers):
|
|
handlers.append((NpmRecipeHandler(), 60))
|