MIRRORS/poky
1
0
Fork 0
mirror of git://git.yoctoproject.org/poky.git synced 2025-07-19 21:09:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity
ffae400179
poky/scripts/lib/recipetool/create_npm.py
Richard Purdie ffae400179 meta/lib+scripts: Convert to SPDX license headers 
This adds SPDX license headers in place of the wide assortment of things
currently in our script headers. We default to GPL-2.0-only except for the
oeqa code where it was clearly submitted and marked as MIT on the most part
or some scripts which had the "or later" GPL versioning.

The patch also drops other obsolete bits of file headers where they were
encoountered such as editor modelines, obsolete maintainer information or
the phrase "All rights reserved" which is now obsolete and not required in
copyright headers (in this case its actually confusing for licensing as all
rights were not reserved).

More work is needed for OE-Core but this takes care of the bulk of the scripts
and meta/lib directories.

The top level LICENSE files are tweaked to match the new structure and the
SPDX naming.

(From OE-Core rev: f8c9c511b5f1b7dbd45b77f345cb6c048ae6763e)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2019-05-09 16:31:55 +01:00

322 lines
14 KiB
Python
Raw Blame History

# Recipe creation tool - node.js NPM module support plugin
#
# Copyright (C) 2016 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
import os
import sys
import logging
import subprocess
import tempfile
import shutil
import json
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars
logger = logging.getLogger('recipetool')
tinfoil = None
def tinfoil_init(instance):
global tinfoil
tinfoil = instance
class NpmRecipeHandler(RecipeHandler):
lockdownpath = None
def _ensure_npm(self, fixed_setup=False):
if not tinfoil.recipes_parsed:
tinfoil.parse_recipes()
try:
rd = tinfoil.parse_recipe('nodejs-native')
except bb.providers.NoProvider:
if fixed_setup:
msg = 'nodejs-native is required for npm but is not available within this SDK'
else:
msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
logger.error(msg)
return None
bindir = rd.getVar('STAGING_BINDIR_NATIVE')
npmpath = os.path.join(bindir, 'npm')
if not os.path.exists(npmpath):
tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
if not os.path.exists(npmpath):
logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
return None
return bindir
def _handle_license(self, data):
'''
Handle the license value from an npm package.json file
'''
license = None
if 'license' in data:
license = data['license']
if isinstance(license, dict):
license = license.get('type', None)
if license:
if 'OR' in license:
license = license.replace('OR', '|')
license = license.replace('AND', '&')
license = license.replace(' ', '_')
if not license[0] == '(':
license = '(' + license + ')'
else:
license = license.replace('AND', '&')
if license[0] == '(':
license = license[1:]
if license[-1] == ')':
license = license[:-1]
license = license.replace('MIT/X11', 'MIT')
license = license.replace('Public Domain', 'PD')
license = license.replace('SEE LICENSE IN EULA',
'SEE-LICENSE-IN-EULA')
return license
def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
try:
runenv = dict(os.environ, PATH=d.getVar('PATH'))
bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
except bb.process.ExecutionError as e:
logger.warning('npm shrinkwrap failed:\n%s' % e.stdout)
return
tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
extravalues.setdefault('extrafiles', {})
extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
runenv = dict(os.environ, PATH=d.getVar('PATH'))
if not NpmRecipeHandler.lockdownpath:
NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
if not os.path.exists(relockbin):
logger.warning('Could not find relock.js within lockdown directory; skipping lockdown')
return
try:
bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
except bb.process.ExecutionError as e:
logger.warning('lockdown-relock failed:\n%s' % e.stdout)
return
tmpfile = os.path.join(localfilesdir, 'lockdown.json')
shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
extravalues.setdefault('extrafiles', {})
extravalues['extrafiles']['lockdown.json'] = tmpfile
lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
import scriptutils
# If this isn't a single module we need to get the dependencies
# and add them to SRC_URI
def varfunc(varname, origvalue, op, newlines):
if varname == 'SRC_URI':
if not origvalue.startswith('npm://'):
src_uri = origvalue.split()
deplist = {}
for dep, depver in optdeps.items():
depdata = self.get_npm_data(dep, depver, d)
if self.check_npm_optional_dependency(depdata):
deplist[dep] = depdata
for dep, depver in devdeps.items():
depdata = self.get_npm_data(dep, depver, d)
if self.check_npm_optional_dependency(depdata):
deplist[dep] = depdata
for dep, depver in deps.items():
depdata = self.get_npm_data(dep, depver, d)
deplist[dep] = depdata
extra_urls = []
for dep, depdata in deplist.items():
version = depdata.get('version', None)
if version:
url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
extra_urls.append(url)
if extra_urls:
scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
src_uri.extend(extra_urls)
return src_uri, None, -1, True
return origvalue, None, 0, True
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
if updated:
del lines_before[:]
for line in newlines:
# Hack to avoid newlines that edit_metadata inserts
if line.endswith('\n'):
line = line[:-1]
lines_before.append(line)
return updated
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
import bb.utils
import oe.package
from collections import OrderedDict
if 'buildsystem' in handled:
return False
def read_package_json(fn):
with open(fn, 'r', errors='surrogateescape') as f:
return json.loads(f.read())
files = RecipeHandler.checkfiles(srctree, ['package.json'])
if files:
d = bb.data.createCopy(tinfoil.config_data)
npm_bindir = self._ensure_npm()
if not npm_bindir:
sys.exit(14)
d.prependVar('PATH', '%s:' % npm_bindir)
data = read_package_json(files[0])
if 'name' in data and 'version' in data:
extravalues['PN'] = data['name']
extravalues['PV'] = data['version']
classes.append('npm')
handled.append('buildsystem')
if 'description' in data:
extravalues['SUMMARY'] = data['description']
if 'homepage' in data:
extravalues['HOMEPAGE'] = data['homepage']
fetchdev = extravalues['fetchdev'] or None
deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
# Shrinkwrap
localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
# Lockdown
self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
# Split each npm module out to is own package
npmpackages = oe.package.npm_split_package_dirs(srctree)
licvalues = None
for item in handled:
if isinstance(item, tuple):
if item[0] == 'license':
licvalues = item[1]
break
if not licvalues:
licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
if licvalues:
# Augment the license list with information we have in the packages
licenses = {}
license = self._handle_license(data)
if license:
licenses['${PN}'] = license
for pkgname, pkgitem in npmpackages.items():
_, pdata = pkgitem
license = self._handle_license(pdata)
if license:
licenses[pkgname] = license
# Now write out the package-specific license values
# We need to strip out the json data dicts for this since split_pkg_licenses
# isn't expecting it
packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
packages['${PN}'] = ''
pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
if '&' in all_licenses:
all_licenses.remove('&')
extravalues['LICENSE'] = ' & '.join(all_licenses)
# Need to move S setting after inherit npm
for i, line in enumerate(lines_before):
if line.startswith('S ='):
lines_before.pop(i)
lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
lines_after.insert(1, line)
break
return True
return False
# FIXME this is duplicated from lib/bb/fetch2/npm.py
def _parse_view(self, output):
'''
Parse the output of npm view --json; the last JSON result
is assumed to be the one that we're interested in.
'''
pdata = None
outdeps = {}
datalines = []
bracelevel = 0
for line in output.splitlines():
if bracelevel:
datalines.append(line)
elif '{' in line:
datalines = []
datalines.append(line)
bracelevel = bracelevel + line.count('{') - line.count('}')
if datalines:
pdata = json.loads('\n'.join(datalines))
return pdata
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
def get_npm_data(self, pkg, version, d):
import bb.fetch2
pkgfullname = pkg
if version != '*' and not '/' in version:
pkgfullname += "@'%s'" % version
logger.debug(2, "Calling getdeps on %s" % pkg)
runenv = dict(os.environ, PATH=d.getVar('PATH'))
fetchcmd = "npm view %s --json" % pkgfullname
output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
data = self._parse_view(output)
return data
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
def get_npm_package_dependencies(self, pdata, fetchdev):
dependencies = pdata.get('dependencies', {})
optionalDependencies = pdata.get('optionalDependencies', {})
dependencies.update(optionalDependencies)
if fetchdev:
devDependencies = pdata.get('devDependencies', {})
dependencies.update(devDependencies)
else:
devDependencies = {}
depsfound = {}
optdepsfound = {}
devdepsfound = {}
for dep in dependencies:
if dep in optionalDependencies:
optdepsfound[dep] = dependencies[dep]
elif dep in devDependencies:
devdepsfound[dep] = dependencies[dep]
else:
depsfound[dep] = dependencies[dep]
return depsfound, optdepsfound, devdepsfound
# FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
# (split out from _getdependencies())
def check_npm_optional_dependency(self, pdata):
pkg_os = pdata.get('os', None)
if pkg_os:
if not isinstance(pkg_os, list):
pkg_os = [pkg_os]
blacklist = False
for item in pkg_os:
if item.startswith('!'):
blacklist = True
break
if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
pkg = pdata.get('name', 'Unnamed package')
logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
return False
return True
def register_recipe_handlers(handlers):
handlers.append((NpmRecipeHandler(), 60))
Reference in New Issue View Git Blame Copy Permalink