poky/scripts/lib/recipetool/create_npm.py

# Recipe creation tool - node.js NPM module support plugin
#
# Copyright (C) 2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import os
import logging
import subprocess
import tempfile
import shutil
import json
from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars

logger = logging.getLogger('recipetool')


tinfoil = None

def tinfoil_init(instance):
    global tinfoil
    tinfoil = instance


class NpmRecipeHandler(RecipeHandler):
    lockdownpath = None

    def _ensure_npm(self, fixed_setup=False):
        if not tinfoil.recipes_parsed:
            tinfoil.parse_recipes()
        try:
            rd = tinfoil.parse_recipe('nodejs-native')
        except bb.providers.NoProvider:
            if fixed_setup:
                msg = 'nodejs-native is required for npm but is not available within this SDK'
            else:
                msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
            logger.error(msg)
            return None
        bindir = rd.getVar('STAGING_BINDIR_NATIVE')
        npmpath = os.path.join(bindir, 'npm')
        if not os.path.exists(npmpath):
            tinfoil.build_targets('nodejs-native', 'addto_recipe_sysroot')
            if not os.path.exists(npmpath):
                logger.error('npm required to process specified source, but nodejs-native did not seem to populate it')
                return None
        return bindir

    def _handle_license(self, data):
        '''
        Handle the license value from an npm package.json file
        '''
        license = None
        if 'license' in data:
            license = data['license']
            if isinstance(license, dict):
                license = license.get('type', None)
            if license:
                if 'OR' in license:
                    license = license.replace('OR', '|')
                    license = license.replace('AND', '&')
                    license = license.replace(' ', '_')
                    if not license[0] == '(':
                        license = '(' + license + ')'
                    print('LICENSE: {}'.format(license))
                else:
                    license = license.replace('AND', '&')
                    if license[0] == '(':
                        license = license[1:]
                    if license[-1] == ')':
                        license = license[:-1]
                license = license.replace('MIT/X11', 'MIT')
                license = license.replace('Public Domain', 'PD')
                license = license.replace('SEE LICENSE IN EULA',
                                          'SEE-LICENSE-IN-EULA')
        return license

    def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
        try:
            runenv = dict(os.environ, PATH=d.getVar('PATH'))
            bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
        except bb.process.ExecutionError as e:
            logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
            return

        tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
        shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
        extravalues.setdefault('extrafiles', {})
        extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
        lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')

    def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
        runenv = dict(os.environ, PATH=d.getVar('PATH'))
        if not NpmRecipeHandler.lockdownpath:
            NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
            bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
                           cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
        relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
        if not os.path.exists(relockbin):
            logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
            return
        try:
            bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
        except bb.process.ExecutionError as e:
            logger.warn('lockdown-relock failed:\n%s' % e.stdout)
            return

        tmpfile = os.path.join(localfilesdir, 'lockdown.json')
        shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
        extravalues.setdefault('extrafiles', {})
        extravalues['extrafiles']['lockdown.json'] = tmpfile
        lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')

    def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
        import scriptutils
        # If this isn't a single module we need to get the dependencies
        # and add them to SRC_URI
        def varfunc(varname, origvalue, op, newlines):
            if varname == 'SRC_URI':
                if not origvalue.startswith('npm://'):
                    src_uri = origvalue.split()
                    deplist = {}
                    for dep, depver in optdeps.items():
                        depdata = self.get_npm_data(dep, depver, d)
                        if self.check_npm_optional_dependency(depdata):
                            deplist[dep] = depdata
                    for dep, depver in devdeps.items():
                        depdata = self.get_npm_data(dep, depver, d)
                        if self.check_npm_optional_dependency(depdata):
                            deplist[dep] = depdata
                    for dep, depver in deps.items():
                        depdata = self.get_npm_data(dep, depver, d)
                        deplist[dep] = depdata

                    extra_urls = []
                    for dep, depdata in deplist.items():
                        version = depdata.get('version', None)
                        if version:
                            url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
                            extra_urls.append(url)
                    if extra_urls:
                        scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
                        src_uri.extend(extra_urls)
                        return src_uri, None, -1, True
            return origvalue, None, 0, True
        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
        if updated:
            del lines_before[:]
            for line in newlines:
                # Hack to avoid newlines that edit_metadata inserts
                if line.endswith('\n'):
                    line = line[:-1]
                lines_before.append(line)
        return updated

    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        import bb.utils
        import oe.package
        from collections import OrderedDict

        if 'buildsystem' in handled:
            return False

        def read_package_json(fn):
            with open(fn, 'r', errors='surrogateescape') as f:
                return json.loads(f.read())

        files = RecipeHandler.checkfiles(srctree, ['package.json'])
        if files:
            d = bb.data.createCopy(tinfoil.config_data)
            npm_bindir = self._ensure_npm()
            if not npm_bindir:
                sys.exit(14)
            d.prependVar('PATH', '%s:' % npm_bindir)

            data = read_package_json(files[0])
            if 'name' in data and 'version' in data:
                extravalues['PN'] = data['name']
                extravalues['PV'] = data['version']
                classes.append('npm')
                handled.append('buildsystem')
                if 'description' in data:
                    extravalues['SUMMARY'] = data['description']
                if 'homepage' in data:
                    extravalues['HOMEPAGE'] = data['homepage']

                fetchdev = extravalues['fetchdev'] or None
                deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
                self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)

                # Shrinkwrap
                localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
                self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)

                # Lockdown
                self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)

                # Split each npm module out to is own package
                npmpackages = oe.package.npm_split_package_dirs(srctree)
                licvalues = None
                for item in handled:
                    if isinstance(item, tuple):
                        if item[0] == 'license':
                            licvalues = item[1]
                            break
                if not licvalues:
                    licvalues = handle_license_vars(srctree, lines_before, handled, extravalues, d)
                if licvalues:
                    # Augment the license list with information we have in the packages
                    licenses = {}
                    license = self._handle_license(data)
                    if license:
                        licenses['${PN}'] = license
                    for pkgname, pkgitem in npmpackages.items():
                        _, pdata = pkgitem
                        license = self._handle_license(pdata)
                        if license:
                            licenses[pkgname] = license
                    # Now write out the package-specific license values
                    # We need to strip out the json data dicts for this since split_pkg_licenses
                    # isn't expecting it
                    packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
                    packages['${PN}'] = ''
                    pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
                    all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
                    if '&' in all_licenses:
                        all_licenses.remove('&')
                    extravalues['LICENSE'] = ' & '.join(all_licenses)

                # Need to move S setting after inherit npm
                for i, line in enumerate(lines_before):
                    if line.startswith('S ='):
                        lines_before.pop(i)
                        lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
                        lines_after.insert(1, line)
                        break

                return True

        return False

    # FIXME this is duplicated from lib/bb/fetch2/npm.py
    def _parse_view(self, output):
        '''
        Parse the output of npm view --json; the last JSON result
        is assumed to be the one that we're interested in.
        '''
        pdata = None
        outdeps = {}
        datalines = []
        bracelevel = 0
        for line in output.splitlines():
            if bracelevel:
                datalines.append(line)
            elif '{' in line:
                datalines = []
                datalines.append(line)
            bracelevel = bracelevel + line.count('{') - line.count('}')
        if datalines:
            pdata = json.loads('\n'.join(datalines))
        return pdata

    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
    # (split out from _getdependencies())
    def get_npm_data(self, pkg, version, d):
        import bb.fetch2
        pkgfullname = pkg
        if version != '*' and not '/' in version:
            pkgfullname += "@'%s'" % version
        logger.debug(2, "Calling getdeps on %s" % pkg)
        runenv = dict(os.environ, PATH=d.getVar('PATH'))
        fetchcmd = "npm view %s --json" % pkgfullname
        output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
        data = self._parse_view(output)
        return data

    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
    # (split out from _getdependencies())
    def get_npm_package_dependencies(self, pdata, fetchdev):
        dependencies = pdata.get('dependencies', {})
        optionalDependencies = pdata.get('optionalDependencies', {})
        dependencies.update(optionalDependencies)
        if fetchdev:
            devDependencies = pdata.get('devDependencies', {})
            dependencies.update(devDependencies)
        else:
            devDependencies = {}
        depsfound = {}
        optdepsfound = {}
        devdepsfound = {}
        for dep in dependencies:
            if dep in optionalDependencies:
                optdepsfound[dep] = dependencies[dep]
            elif dep in devDependencies:
                devdepsfound[dep] = dependencies[dep]
            else:
                depsfound[dep] = dependencies[dep]
        return depsfound, optdepsfound, devdepsfound

    # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
    # (split out from _getdependencies())
    def check_npm_optional_dependency(self, pdata):
        pkg_os = pdata.get('os', None)
        if pkg_os:
            if not isinstance(pkg_os, list):
                pkg_os = [pkg_os]
            blacklist = False
            for item in pkg_os:
                if item.startswith('!'):
                    blacklist = True
                    break
            if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
                logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
                return False
        return True


def register_recipe_handlers(handlers):
    handlers.append((NpmRecipeHandler(), 60))