mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 12:59:02 +02:00
recipetool/create_npm: refactor the npm recipe creation handler
This commit refactors the npm recipe creation handler to use the new npm behavior. The process is kept as simple as possible and only generates the shrinkwrap file. To avoid naming issues the recipe name is now extracted from the npm package name and not directly mapped. (From OE-Core rev: 1deccb0f0c204cd02fb8606f180d8a13df9f31db) Signed-off-by: Jean-Marie LEMETAYER <jean-marie.lemetayer@savoirfairelinux.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Jean-Marie LEMETAYER
Richard Purdie
parent
3a8a675825
commit
6fd9cebc98
|
|
@ -1,321 +1,209 @@
|
|||
# Recipe creation tool - node.js NPM module support plugin
|
||||
#
|
||||
# Copyright (C) 2016 Intel Corporation
|
||||
# Copyright (C) 2020 Savoir-Faire Linux
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
#
|
||||
"""Recipe creation tool - npm module support plugin"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import subprocess
|
||||
import tempfile
|
||||
import shutil
|
||||
import json
|
||||
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import bb
|
||||
from bb.fetch2.npm import NpmEnvironment
|
||||
from recipetool.create import RecipeHandler
|
||||
|
||||
logger = logging.getLogger('recipetool')
|
||||
|
||||
|
||||
tinfoil = None
|
||||
TINFOIL = None
|
||||
|
||||
def tinfoil_init(instance):
|
||||
global tinfoil
|
||||
tinfoil = instance
|
||||
|
||||
"""Initialize tinfoil"""
|
||||
global TINFOIL
|
||||
TINFOIL = instance
|
||||
|
||||
class NpmRecipeHandler(RecipeHandler):
|
||||
lockdownpath = None
|
||||
"""Class to handle the npm recipe creation"""
|
||||
|
||||
@staticmethod
|
||||
def _npm_name(name):
|
||||
"""Generate a Yocto friendly npm name"""
|
||||
name = re.sub("/", "-", name)
|
||||
name = name.lower()
|
||||
name = re.sub(r"[^\-a-z0-9]", "", name)
|
||||
name = name.strip("-")
|
||||
return name
|
||||
|
||||
@staticmethod
|
||||
def _get_registry(lines):
|
||||
"""Get the registry value from the 'npm://registry' url"""
|
||||
registry = None
|
||||
|
||||
def _handle_registry(varname, origvalue, op, newlines):
|
||||
nonlocal registry
|
||||
if origvalue.startswith("npm://"):
|
||||
registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
|
||||
return origvalue, None, 0, True
|
||||
|
||||
bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
|
||||
|
||||
return registry
|
||||
|
||||
@staticmethod
|
||||
def _ensure_npm():
|
||||
"""Check if the 'npm' command is available in the recipes"""
|
||||
if not TINFOIL.recipes_parsed:
|
||||
TINFOIL.parse_recipes()
|
||||
|
||||
def _ensure_npm(self, fixed_setup=False):
|
||||
if not tinfoil.recipes_parsed:
|
||||
tinfoil.parse_recipes()
|
||||
try:
|
||||
rd = tinfoil.parse_recipe('nodejs-native')
|
||||
d = TINFOIL.parse_recipe("nodejs-native")
|
||||
except bb.providers.NoProvider:
|
||||
if fixed_setup:
|
||||
msg = 'nodejs-native is required for npm but is not available within this SDK'
|
||||
else:
|
||||
msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
|
||||
logger.error(msg)
|
||||
return None
|
||||
bindir = rd.getVar('STAGING_BINDIR_NATIVE')
|
||||
npmpath = os.path.join(bindir, 'npm')
|
||||
bb.error("Nothing provides 'nodejs-native' which is required for the build")
|
||||
bb.note("You will likely need to add a layer that provides nodejs")
|
||||
sys.exit(14)
|
||||
|
||||
bindir = d.getVar("STAGING_BINDIR_NATIVE")
|
||||
npmpath = os.path.join(bindir, "npm")
|
||||
|
||||
if not os.path.exists(npmpath):
|
||||
tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
|
||||
TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
|
||||
|
||||
if not os.path.exists(npmpath):
|
||||
logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
|
||||
return None
|
||||
bb.error("Failed to add 'npm' to sysroot")
|
||||
sys.exit(14)
|
||||
|
||||
return bindir
|
||||
|
||||
def _handle_license(self, data):
|
||||
'''
|
||||
Handle the license value from an npm package.json file
|
||||
'''
|
||||
license = None
|
||||
if 'license' in data:
|
||||
license = data['license']
|
||||
if isinstance(license, dict):
|
||||
license = license.get('type', None)
|
||||
if license:
|
||||
if 'OR' in license:
|
||||
license = license.replace('OR', '|')
|
||||
license = license.replace('AND', '&')
|
||||
license = license.replace(' ', '_')
|
||||
if not license[0] == '(':
|
||||
license = '(' + license + ')'
|
||||
else:
|
||||
license = license.replace('AND', '&')
|
||||
if license[0] == '(':
|
||||
license = license[1:]
|
||||
if license[-1] == ')':
|
||||
license = license[:-1]
|
||||
license = license.replace('MIT/X11', 'MIT')
|
||||
license = license.replace('Public Domain', 'PD')
|
||||
license = license.replace('SEE LICENSE IN EULA',
|
||||
'SEE-LICENSE-IN-EULA')
|
||||
return license
|
||||
@staticmethod
|
||||
def _npm_global_configs(dev):
|
||||
"""Get the npm global configuration"""
|
||||
configs = []
|
||||
|
||||
def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
|
||||
try:
|
||||
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
||||
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||||
except bb.process.ExecutionError as e:
|
||||
logger.warning('npm shrinkwrap failed:\n%s' % e.stdout)
|
||||
return
|
||||
if dev:
|
||||
configs.append(("also", "development"))
|
||||
else:
|
||||
configs.append(("only", "production"))
|
||||
|
||||
tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
|
||||
shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
|
||||
extravalues.setdefault('extrafiles', {})
|
||||
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
|
||||
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
|
||||
configs.append(("save", "false"))
|
||||
configs.append(("package-lock", "false"))
|
||||
configs.append(("shrinkwrap", "false"))
|
||||
return configs
|
||||
|
||||
def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
|
||||
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
||||
if not NpmRecipeHandler.lockdownpath:
|
||||
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
|
||||
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
|
||||
cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||||
relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
|
||||
if not os.path.exists(relockbin):
|
||||
logger.warning('Could not find relock.js within lockdown directory; skipping lockdown')
|
||||
return
|
||||
try:
|
||||
bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||||
except bb.process.ExecutionError as e:
|
||||
logger.warning('lockdown-relock failed:\n%s' % e.stdout)
|
||||
return
|
||||
def _run_npm_install(self, d, srctree, registry, dev):
|
||||
"""Run the 'npm install' command without building the addons"""
|
||||
configs = self._npm_global_configs(dev)
|
||||
configs.append(("ignore-scripts", "true"))
|
||||
|
||||
tmpfile = os.path.join(localfilesdir, 'lockdown.json')
|
||||
shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
|
||||
extravalues.setdefault('extrafiles', {})
|
||||
extravalues['extrafiles']['lockdown.json'] = tmpfile
|
||||
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
|
||||
if registry:
|
||||
configs.append(("registry", registry))
|
||||
|
||||
def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
|
||||
import scriptutils
|
||||
# If this isn't a single module we need to get the dependencies
|
||||
# and add them to SRC_URI
|
||||
def varfunc(varname, origvalue, op, newlines):
|
||||
if varname == 'SRC_URI':
|
||||
if not origvalue.startswith('npm://'):
|
||||
src_uri = origvalue.split()
|
||||
deplist = {}
|
||||
for dep, depver in optdeps.items():
|
||||
depdata = self.get_npm_data(dep, depver, d)
|
||||
if self.check_npm_optional_dependency(depdata):
|
||||
deplist[dep] = depdata
|
||||
for dep, depver in devdeps.items():
|
||||
depdata = self.get_npm_data(dep, depver, d)
|
||||
if self.check_npm_optional_dependency(depdata):
|
||||
deplist[dep] = depdata
|
||||
for dep, depver in deps.items():
|
||||
depdata = self.get_npm_data(dep, depver, d)
|
||||
deplist[dep] = depdata
|
||||
bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
|
||||
|
||||
extra_urls = []
|
||||
for dep, depdata in deplist.items():
|
||||
version = depdata.get('version', None)
|
||||
if version:
|
||||
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
|
||||
extra_urls.append(url)
|
||||
if extra_urls:
|
||||
scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
|
||||
src_uri.extend(extra_urls)
|
||||
return src_uri, None, -1, True
|
||||
return origvalue, None, 0, True
|
||||
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
|
||||
if updated:
|
||||
del lines_before[:]
|
||||
for line in newlines:
|
||||
# Hack to avoid newlines that edit_metadata inserts
|
||||
if line.endswith('\n'):
|
||||
line = line[:-1]
|
||||
lines_before.append(line)
|
||||
return updated
|
||||
env = NpmEnvironment(d, configs=configs)
|
||||
env.run("npm install", workdir=srctree)
|
||||
|
||||
def _generate_shrinkwrap(self, d, srctree, dev):
|
||||
"""Check and generate the 'npm-shrinkwrap.json' file if needed"""
|
||||
configs = self._npm_global_configs(dev)
|
||||
|
||||
env = NpmEnvironment(d, configs=configs)
|
||||
env.run("npm shrinkwrap", workdir=srctree)
|
||||
|
||||
return os.path.join(srctree, "npm-shrinkwrap.json")
|
||||
|
||||
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
||||
import bb.utils
|
||||
import oe.package
|
||||
from collections import OrderedDict
|
||||
"""Handle the npm recipe creation"""
|
||||
|
||||
if 'buildsystem' in handled:
|
||||
if "buildsystem" in handled:
|
||||
return False
|
||||
|
||||
def read_package_json(fn):
|
||||
with open(fn, 'r', errors='surrogateescape') as f:
|
||||
return json.loads(f.read())
|
||||
files = RecipeHandler.checkfiles(srctree, ["package.json"])
|
||||
|
||||
files = RecipeHandler.checkfiles(srctree, ['package.json'])
|
||||
if files:
|
||||
d = bb.data.createCopy(tinfoil.config_data)
|
||||
npm_bindir = self._ensure_npm()
|
||||
if not npm_bindir:
|
||||
sys.exit(14)
|
||||
d.prependVar('PATH', '%s:' % npm_bindir)
|
||||
if not files:
|
||||
return False
|
||||
|
||||
data = read_package_json(files[0])
|
||||
if 'name' in data and 'version' in data:
|
||||
extravalues['PN'] = data['name']
|
||||
extravalues['PV'] = data['version']
|
||||
classes.append('npm')
|
||||
handled.append('buildsystem')
|
||||
if 'description' in data:
|
||||
extravalues['SUMMARY'] = data['description']
|
||||
if 'homepage' in data:
|
||||
extravalues['HOMEPAGE'] = data['homepage']
|
||||
with open(files[0], "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
fetchdev = extravalues['fetchdev'] or None
|
||||
deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
|
||||
self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
|
||||
if "name" not in data or "version" not in data:
|
||||
return False
|
||||
|
||||
# Shrinkwrap
|
||||
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
|
||||
self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
|
||||
extravalues["PN"] = self._npm_name(data["name"])
|
||||
extravalues["PV"] = data["version"]
|
||||
|
||||
# Lockdown
|
||||
self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
|
||||
if "description" in data:
|
||||
extravalues["SUMMARY"] = data["description"]
|
||||
|
||||
# Split each npm module out to is own package
|
||||
npmpackages = oe.package.npm_split_package_dirs(srctree)
|
||||
licvalues = None
|
||||
for item in handled:
|
||||
if isinstance(item, tuple):
|
||||
if item[0] == 'license':
|
||||
licvalues = item[1]
|
||||
break
|
||||
if not licvalues:
|
||||
licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
|
||||
if licvalues:
|
||||
# Augment the license list with information we have in the packages
|
||||
licenses = {}
|
||||
license = self._handle_license(data)
|
||||
if license:
|
||||
licenses['${PN}'] = license
|
||||
for pkgname, pkgitem in npmpackages.items():
|
||||
_, pdata = pkgitem
|
||||
license = self._handle_license(pdata)
|
||||
if license:
|
||||
licenses[pkgname] = license
|
||||
# Now write out the package-specific license values
|
||||
# We need to strip out the json data dicts for this since split_pkg_licenses
|
||||
# isn't expecting it
|
||||
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
|
||||
packages['${PN}'] = ''
|
||||
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
|
||||
all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
|
||||
if '&' in all_licenses:
|
||||
all_licenses.remove('&')
|
||||
extravalues['LICENSE'] = ' & '.join(all_licenses)
|
||||
if "homepage" in data:
|
||||
extravalues["HOMEPAGE"] = data["homepage"]
|
||||
|
||||
# Need to move S setting after inherit npm
|
||||
for i, line in enumerate(lines_before):
|
||||
if line.startswith('S ='):
|
||||
lines_before.pop(i)
|
||||
lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
|
||||
lines_after.insert(1, line)
|
||||
break
|
||||
dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
|
||||
registry = self._get_registry(lines_before)
|
||||
|
||||
return True
|
||||
bb.note("Checking if npm is available ...")
|
||||
# The native npm is used here (and not the host one) to ensure that the
|
||||
# npm version is high enough to ensure an efficient dependency tree
|
||||
# resolution and avoid issue with the shrinkwrap file format.
|
||||
# Moreover the native npm is mandatory for the build.
|
||||
bindir = self._ensure_npm()
|
||||
|
||||
return False
|
||||
d = bb.data.createCopy(TINFOIL.config_data)
|
||||
d.prependVar("PATH", bindir + ":")
|
||||
d.setVar("S", srctree)
|
||||
|
||||
# FIXME this is duplicated from lib/bb/fetch2/npm.py
|
||||
def _parse_view(self, output):
|
||||
'''
|
||||
Parse the output of npm view --json; the last JSON result
|
||||
is assumed to be the one that we're interested in.
|
||||
'''
|
||||
pdata = None
|
||||
outdeps = {}
|
||||
datalines = []
|
||||
bracelevel = 0
|
||||
for line in output.splitlines():
|
||||
if bracelevel:
|
||||
datalines.append(line)
|
||||
elif '{' in line:
|
||||
datalines = []
|
||||
datalines.append(line)
|
||||
bracelevel = bracelevel + line.count('{') - line.count('}')
|
||||
if datalines:
|
||||
pdata = json.loads('\n'.join(datalines))
|
||||
return pdata
|
||||
bb.note("Generating shrinkwrap file ...")
|
||||
# To generate the shrinkwrap file the dependencies have to be installed
|
||||
# first. During the generation process some files may be updated /
|
||||
# deleted. By default devtool tracks the diffs in the srctree and raises
|
||||
# errors when finishing the recipe if some diffs are found.
|
||||
git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
|
||||
if os.path.exists(git_exclude_file):
|
||||
with open(git_exclude_file, "r+") as f:
|
||||
lines = f.readlines()
|
||||
for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
|
||||
if line not in lines:
|
||||
f.write(line + "\n")
|
||||
|
||||
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
||||
# (split out from _getdependencies())
|
||||
def get_npm_data(self, pkg, version, d):
|
||||
import bb.fetch2
|
||||
pkgfullname = pkg
|
||||
if version != '*' and not '/' in version:
|
||||
pkgfullname += "@'%s'" % version
|
||||
logger.debug(2, "Calling getdeps on %s" % pkg)
|
||||
runenv = dict(os.environ, PATH=d.getVar('PATH'))
|
||||
fetchcmd = "npm view %s --json" % pkgfullname
|
||||
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
|
||||
data = self._parse_view(output)
|
||||
return data
|
||||
lock_file = os.path.join(srctree, "package-lock.json")
|
||||
lock_copy = lock_file + ".copy"
|
||||
if os.path.exists(lock_file):
|
||||
bb.utils.copyfile(lock_file, lock_copy)
|
||||
|
||||
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
||||
# (split out from _getdependencies())
|
||||
def get_npm_package_dependencies(self, pdata, fetchdev):
|
||||
dependencies = pdata.get('dependencies', {})
|
||||
optionalDependencies = pdata.get('optionalDependencies', {})
|
||||
dependencies.update(optionalDependencies)
|
||||
if fetchdev:
|
||||
devDependencies = pdata.get('devDependencies', {})
|
||||
dependencies.update(devDependencies)
|
||||
else:
|
||||
devDependencies = {}
|
||||
depsfound = {}
|
||||
optdepsfound = {}
|
||||
devdepsfound = {}
|
||||
for dep in dependencies:
|
||||
if dep in optionalDependencies:
|
||||
optdepsfound[dep] = dependencies[dep]
|
||||
elif dep in devDependencies:
|
||||
devdepsfound[dep] = dependencies[dep]
|
||||
else:
|
||||
depsfound[dep] = dependencies[dep]
|
||||
return depsfound, optdepsfound, devdepsfound
|
||||
self._run_npm_install(d, srctree, registry, dev)
|
||||
shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
|
||||
|
||||
if os.path.exists(lock_copy):
|
||||
bb.utils.movefile(lock_copy, lock_file)
|
||||
|
||||
# Add the shrinkwrap file as 'extrafiles'
|
||||
shrinkwrap_copy = shrinkwrap_file + ".copy"
|
||||
bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
|
||||
extravalues.setdefault("extrafiles", {})
|
||||
extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
|
||||
|
||||
url_local = "npmsw://%s" % shrinkwrap_file
|
||||
url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
|
||||
|
||||
if dev:
|
||||
url_local += ";dev=1"
|
||||
url_recipe += ";dev=1"
|
||||
|
||||
# Add the npmsw url in the SRC_URI of the generated recipe
|
||||
def _handle_srcuri(varname, origvalue, op, newlines):
|
||||
"""Update the version value and add the 'npmsw://' url"""
|
||||
value = origvalue.replace("version=" + data["version"], "version=${PV}")
|
||||
value = value.replace("version=latest", "version=${PV}")
|
||||
values = [line.strip() for line in value.strip('\n').splitlines()]
|
||||
values.append(url_recipe)
|
||||
return values, None, 4, False
|
||||
|
||||
(_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
|
||||
lines_before[:] = [line.rstrip('\n') for line in newlines]
|
||||
|
||||
classes.append("npm")
|
||||
handled.append("buildsystem")
|
||||
|
||||
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
|
||||
# (split out from _getdependencies())
|
||||
def check_npm_optional_dependency(self, pdata):
|
||||
pkg_os = pdata.get('os', None)
|
||||
if pkg_os:
|
||||
if not isinstance(pkg_os, list):
|
||||
pkg_os = [pkg_os]
|
||||
blacklist = False
|
||||
for item in pkg_os:
|
||||
if item.startswith('!'):
|
||||
blacklist = True
|
||||
break
|
||||
if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
|
||||
pkg = pdata.get('name', 'Unnamed package')
|
||||
logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def register_recipe_handlers(handlers):
|
||||
"""Register the npm handler"""
|
||||
handlers.append((NpmRecipeHandler(), 60))
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user