MIRRORS/poky
1
0
Fork 0
mirror of git://git.yoctoproject.org/poky.git synced 2025-07-19 12:59:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

classes/npm: refactor the npm class

Browse Source 
This commit splits the npm build in three steps:

1. With the new npmsw fetcher, the sources and dependencies of the
   package have been fetched and unpacked. As sources can also be
   patched, a local cache must be configured to use these modified
   sources.

2. Next, the installation process is run using the local cache only.
   Some packages may need to be compiled.

3. The final installation filters the previously installed files to
   avoid unnecessary files.

This new version also fixes multiple issues related to npm dependencies
badly handled: package names, scope packages, installation directories

(From OE-Core rev: fb2252ee0777c6d26dea94c7588c323a6b97e961)

Signed-off-by: Jean-Marie LEMETAYER <jean-marie.lemetayer@savoirfairelinux.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Jean-Marie LEMETAYER 2020-01-24 18:07:31 +01:00 committed by Richard Purdie
parent 97c882303e
commit de39f14d24
1 changed files with 278 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

357
meta/classes/npm.bbclass
View File

@ -1,94 +1,293 @@
# Copyright (C) 2020 Savoir-Faire Linux
#
# SPDX-License-Identifier: GPL-2.0-only
#
# This bbclass builds and installs an npm package to the target. The package
# sources files should be fetched in the calling recipe by using the SRC_URI
# variable. The ${S} variable should be updated depending of your fetcher.
#
# Usage:
# SRC_URI = "..."
# inherit npm
#
# Optional variables:
# NPM_ARCH:
# Override the auto generated npm architecture.
#
# NPM_INSTALL_DEV:
# Set to 1 to also install devDependencies.
DEPENDS_prepend = "nodejs-native "
RDEPENDS_${PN}_prepend = "nodejs "
S = "${WORKDIR}/npmpkg"
def node_pkgname(d):
bpn = d.getVar('BPN')
if bpn.startswith("node-"):
return bpn[5:]
return bpn
NPMPN ?= "${@node_pkgname(d)}"
NPM_INSTALLDIR = "${libdir}/node_modules/${NPMPN}"
# function maps arch names to npm arch names
def npm_oe_arch_map(target_arch, d):
import re
if re.match('p(pc|owerpc)(|64)', target_arch): return 'ppc'
elif re.match('i.86$', target_arch): return 'ia32'
elif re.match('x86_64$', target_arch): return 'x64'
elif re.match('arm64$', target_arch): return 'arm'
return target_arch
NPM_ARCH ?= "${@npm_oe_arch_map(d.getVar('TARGET_ARCH'), d)}"
NPM_INSTALL_DEV ?= "0"
npm_do_compile() {
# Copy in any additionally fetched modules
if [ -d ${WORKDIR}/node_modules ] ; then
cp -a ${WORKDIR}/node_modules ${S}/
fi
# changing the home directory to the working directory, the .npmrc will
# be created in this directory
export HOME=${WORKDIR}
if [ "${NPM_INSTALL_DEV}" = "1" ]; then
npm config set dev true
else
npm config set dev false
fi
npm set cache ${WORKDIR}/npm_cache
# clear cache before every build
npm cache clear --force
# Install pkg into ${S} without going to the registry
if [ "${NPM_INSTALL_DEV}" = "1" ]; then
npm --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --no-registry install
else
npm --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry install
fi
def npm_target_arch_map(target_arch):
"""Maps arch names to npm arch names"""
import re
if re.match("p(pc|owerpc)(|64)", target_arch):
return "ppc"
elif re.match("i.86$", target_arch):
return "ia32"
elif re.match("x86_64$", target_arch):
return "x64"
elif re.match("arm64$", target_arch):
return "arm"
return target_arch
NPM_ARCH ?= "${@npm_target_arch_map(d.getVar("TARGET_ARCH"))}"
NPM_PACKAGE = "${WORKDIR}/npm-package"
NPM_CACHE = "${WORKDIR}/npm-cache"
NPM_BUILD = "${WORKDIR}/npm-build"
def npm_global_configs(d):
"""Get the npm global configuration"""
configs = []
# Configure the cache directory
configs.append(("cache", d.getVar("NPM_CACHE")))
return configs
def npm_pack(env, srcdir, workdir):
"""Run 'npm pack' on a specified directory"""
import shlex
cmd = "npm pack %s" % shlex.quote(srcdir)
configs = [("ignore-scripts", "true")]
tarball = env.run(cmd, configs=configs, workdir=workdir).strip("\n")
return os.path.join(workdir, tarball)
python npm_do_configure() {
"""
Step one: configure the npm cache and the main npm package
Every dependencies have been fetched and patched in the source directory.
They have to be packed (this remove unneeded files) and added to the npm
cache to be available for the next step.
The main package and its associated manifest file and shrinkwrap file have
to be configured to take into account these cached dependencies.
"""
import base64
import copy
import json
import re
import shlex
import tempfile
from bb.fetch2.npm import NpmEnvironment
from bb.fetch2.npm import npm_unpack
from bb.fetch2.npmsw import foreach_dependencies
from bb.progress import OutOfProgressHandler
bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
env = NpmEnvironment(d, configs=npm_global_configs(d))
def _npm_cache_add(tarball):
"""Run 'npm cache add' for a specified tarball"""
cmd = "npm cache add %s" % shlex.quote(tarball)
env.run(cmd)
def _npm_integrity(tarball):
"""Return the npm integrity of a specified tarball"""
sha512 = bb.utils.sha512_file(tarball)
return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode()
def _npm_version(tarball):
"""Return the version of a specified tarball"""
regex = r"-(\d+\.\d+\.\d+(-.*)?(\+.*)?)\.tgz"
return re.search(regex, tarball).group(1)
def _npmsw_dependency_dict(orig, deptree):
"""
Return the sub dictionary in the 'orig' dictionary corresponding to the
'deptree' dependency tree. This function follows the shrinkwrap file
format.
"""
ptr = orig
for dep in deptree:
if "dependencies" not in ptr:
ptr["dependencies"] = {}
ptr = ptr["dependencies"]
if dep not in ptr:
ptr[dep] = {}
ptr = ptr[dep]
return ptr
# Manage the manifest file and shrinkwrap files
orig_manifest_file = d.expand("${S}/package.json")
orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json")
cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json")
cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json")
with open(orig_manifest_file, "r") as f:
orig_manifest = json.load(f)
cached_manifest = copy.deepcopy(orig_manifest)
cached_manifest.pop("dependencies", None)
cached_manifest.pop("devDependencies", None)
with open(orig_shrinkwrap_file, "r") as f:
orig_shrinkwrap = json.load(f)
cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
cached_shrinkwrap.pop("dependencies", None)
# Manage the dependencies
progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
progress_total = 1 # also count the main package
progress_done = 0
def _count_dependency(name, params, deptree):
nonlocal progress_total
progress_total += 1
def _cache_dependency(name, params, deptree):
destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
destsuffix = os.path.join(*destsubdirs)
with tempfile.TemporaryDirectory() as tmpdir:
# Add the dependency to the npm cache
destdir = os.path.join(d.getVar("S"), destsuffix)
tarball = npm_pack(env, destdir, tmpdir)
_npm_cache_add(tarball)
# Add its signature to the cached shrinkwrap
dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree)
dep["version"] = _npm_version(tarball)
dep["integrity"] = _npm_integrity(tarball)
if params.get("dev", False):
dep["dev"] = True
# Display progress
nonlocal progress_done
progress_done += 1
progress.write("%d/%d" % (progress_done, progress_total))
dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
# Configure the main package
with tempfile.TemporaryDirectory() as tmpdir:
tarball = npm_pack(env, d.getVar("S"), tmpdir)
npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d)
# Configure the cached manifest file and cached shrinkwrap file
def _update_manifest(depkey):
for name in orig_manifest.get(depkey, {}):
version = cached_shrinkwrap["dependencies"][name]["version"]
if depkey not in cached_manifest:
cached_manifest[depkey] = {}
cached_manifest[depkey][name] = version
_update_manifest("dependencies")
if dev:
_update_manifest("devDependencies")
with open(cached_manifest_file, "w") as f:
json.dump(cached_manifest, f, indent=2)
with open(cached_shrinkwrap_file, "w") as f:
json.dump(cached_shrinkwrap, f, indent=2)
}
python npm_do_compile() {
"""
Step two: install the npm package
Use the configured main package and the cached dependencies to run the
installation process. The installation is done in a directory which is
not the destination directory yet.
A combination of 'npm pack' and 'npm install' is used to ensure that the
installed files are actual copies instead of symbolic links (which is the
default npm behavior).
"""
import shlex
import tempfile
from bb.fetch2.npm import NpmEnvironment
bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)
env = NpmEnvironment(d, configs=npm_global_configs(d))
dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
with tempfile.TemporaryDirectory() as tmpdir:
configs = []
if dev:
configs.append(("also", "development"))
else:
configs.append(("only", "production"))
# Report as many logs as possible for debugging purpose
configs.append(("loglevel", "silly"))
# Configure the installation to be done globally in the build directory
configs.append(("global", "true"))
configs.append(("prefix", d.getVar("NPM_BUILD")))
# Add node-gyp configuration
configs.append(("arch", d.getVar("NPM_ARCH")))
configs.append(("release", "true"))
# Pack and install the main package
tarball = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
env.run("npm install %s" % shlex.quote(tarball), configs=configs)
}
npm_do_install() {
# changing the home directory to the working directory, the .npmrc will
# be created in this directory
export HOME=${WORKDIR}
mkdir -p ${D}${libdir}/node_modules
local NPM_PACKFILE=$(npm pack .)
npm install --prefix ${D}${prefix} -g --arch=${NPM_ARCH} --target_arch=${NPM_ARCH} --production --no-registry ${NPM_PACKFILE}
ln -fs node_modules ${D}${libdir}/node
find ${D}${NPM_INSTALLDIR} -type f \( -name "*.a" -o -name "*.d" -o -name "*.o" \) -delete
if [ -d ${D}${prefix}/etc ] ; then
# This will be empty
rmdir ${D}${prefix}/etc
fi
}
# Step three: final install
#
# The previous installation have to be filtered to remove some extra files.
python populate_packages_prepend () {
instdir = d.expand('${D}${NPM_INSTALLDIR}')
extrapackages = oe.package.npm_split_package_dirs(instdir)
pkgnames = extrapackages.keys()
d.prependVar('PACKAGES', '%s ' % ' '.join(pkgnames))
for pkgname in pkgnames:
pkgrelpath, pdata = extrapackages[pkgname]
pkgpath = '${NPM_INSTALLDIR}/' + pkgrelpath
# package names can't have underscores but npm packages sometimes use them
oe_pkg_name = pkgname.replace('_', '-')
expanded_pkgname = d.expand(oe_pkg_name)
d.setVar('FILES_%s' % expanded_pkgname, pkgpath)
if pdata:
version = pdata.get('version', None)
if version:
d.setVar('PKGV_%s' % expanded_pkgname, version)
description = pdata.get('description', None)
if description:
d.setVar('SUMMARY_%s' % expanded_pkgname, description.replace(u"\u2018", "'").replace(u"\u2019", "'"))
d.appendVar('RDEPENDS_%s' % d.getVar('PN'), ' %s' % ' '.join(pkgnames).replace('_', '-'))
rm -rf ${D}
# Copy the entire lib and bin directories
install -d ${D}/${nonarch_libdir}
cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/. ${D}/${nonarch_libdir}
if [ -d "${NPM_BUILD}/bin" ]
then
install -d ${D}/${bindir}
cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/. ${D}/${bindir}
fi
# If the package (or its dependencies) uses node-gyp to build native addons,
# object files, static libraries or other temporary files can be hidden in
# the lib directory. To reduce the package size and to avoid QA issues
# (staticdev with static library files) these files must be removed.
local GYP_REGEX=".*/build/Release/[^/]*.node"
# Remove any node-gyp directory in ${D} to remove temporary build files
for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}")
do
local GYP_D_DIR=${GYP_D_FILE%/Release/*}
rm --recursive --force ${GYP_D_DIR}
done
# Copy only the node-gyp release files
for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}")
do
local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}}
install -d ${GYP_D_FILE%/*}
install -m 755 ${GYP_B_FILE} ${GYP_D_FILE}
done
# Remove the shrinkwrap file which does not need to be packed
rm -f ${D}/${nonarch_libdir}/node_modules/*/npm-shrinkwrap.json
rm -f ${D}/${nonarch_libdir}/node_modules/@*/*/npm-shrinkwrap.json
# node(1) is using /usr/lib/node as default include directory and npm(1) is
# using /usr/lib/node_modules as install directory. Let's make both happy.
ln -fs node_modules ${D}/${nonarch_libdir}/node
}
FILES_${PN} += " \
${bindir} \
${libdir}/node \
${NPM_INSTALLDIR} \
${nonarch_libdir} \
"
EXPORT_FUNCTIONS do_compile do_install
EXPORT_FUNCTIONS do_configure do_compile do_install