
image-manifest: script to generate product/image specific BOM The image-manifest script generates image specific reports based on an image manifest file. Currently there is data generated by buildhistory, pkgdata, and license manifest but this data is poorly formated and spread across multiple text files. This script can generate a single JSON output file that is machine readable by other tools. The manifest-info collects package information and stores the information in a tarball. manifest-info can be configured using a json configuration file. The default configuration including all possible options can be dumped using the dump-config subcommand. image-manifest takes an image manifest file as input to get the runtime dependencies. As an option image-manifest can also use the build dependency file, pn-buildlist, to get the build dependencies excluding native packages. This script extends the oe-image-manifest script [0] done by Paul Eggleton [0] https://github.com/intel/clear-linux-dissector-web/blob/master/layerindex/static/files/oe-image-manifest ------------------------------------------------------ usage: image-manifest [-h] [-d] [-q] <subcommand> ... Image manifest utility options: -h, --help show this help message and exit -d, --debug Enable debug output -q, --quiet Print only errors subcommands: recipe-info Get recipe info list-depends List dependencies list-recipes List recipes producing packages within an image list-packages List packages within an image list-layers List included layers dump-config Dump default config manifest-info Export recipe info for a manifest Use image-manifest <subcommand> --help to get help on a specific command Co-developed-by: Paul Eggleton <bluelightning@bluelightning.org> (From OE-Core rev: ad8fec9ce1704866df925bda18a240d6889b1ed5) Signed-off-by: Anders Wallin <anders.wallin@windriver.com> Signed-off-by: Saul Wold <saul.wold@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
20 KiB
Executable File
#!/usr/bin/env python3
Script to extract information from image manifests
Copyright (C) 2018 Intel Corporation
Copyright (C) 2021 Wind River Systems, Inc.
SPDX-License-Identifier: GPL-2.0-only
import sys import os import argparse import logging import json import shutil import tempfile import tarfile from collections import OrderedDict
scripts_path = os.path.dirname(file) lib_path = scripts_path + '/../lib' sys.path = sys.path + [lib_path]
import scriptutils logger = scriptutils.logger_create(os.path.basename(file))
import argparse_oe import scriptpath bitbakepath = scriptpath.add_bitbake_lib_path() if not bitbakepath: logger.error("Unable to find bitbake by searching parent directory of this script or PATH") sys.exit(1) logger.debug('Using standard bitbake path %s' % bitbakepath) scriptpath.add_oe_lib_path()
import bb.tinfoil import bb.utils import oe.utils import oe.recipeutils
def get_pkg_list(manifest): pkglist = [] with open(manifest, 'r') as f: for line in f: linesplit = line.split() if len(linesplit) == 3: # manifest file pkglist.append(linesplit[0]) elif len(linesplit) == 1: # build dependency file pkglist.append(linesplit[0]) return sorted(pkglist)
def list_packages(args): pkglist = get_pkg_list(args.manifest) for pkg in pkglist: print('%s' % pkg)
def pkg2recipe(tinfoil, pkg): if "-native" in pkg: logger.info('skipping %s' % pkg) return None
pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
pkgdatafile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
logger.debug('pkgdatafile %s' % pkgdatafile)
try:
f = open(pkgdatafile, 'r')
for line in f:
if line.startswith('PN:'):
recipe = line.split(':', 1)[1].strip()
return recipe
except Exception:
logger.warning('%s is missing' % pkgdatafile)
return None
def get_recipe_list(manifest, tinfoil): pkglist = get_pkg_list(manifest) recipelist = [] for pkg in pkglist: recipe = pkg2recipe(tinfoil,pkg) if recipe: if not recipe in recipelist: recipelist.append(recipe)
return sorted(recipelist)
def list_recipes(args): import bb.tinfoil with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.logger.setLevel(logger.getEffectiveLevel()) tinfoil.prepare(config_only=True) recipelist = get_recipe_list(args.manifest, tinfoil) for recipe in sorted(recipelist): print('%s' % recipe)
def list_layers(args):
def find_git_repo(pth):
checkpth = pth
while checkpth != os.sep:
if os.path.exists(os.path.join(checkpth, '.git')):
return checkpth
checkpth = os.path.dirname(checkpth)
return None
def get_git_remote_branch(repodir):
try:
stdout, _ = bb.process.run(['git', 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{u}'], cwd=repodir)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
return stdout.strip()
else:
return None
def get_git_head_commit(repodir):
try:
stdout, _ = bb.process.run(['git', 'rev-parse', 'HEAD'], cwd=repodir)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
return stdout.strip()
else:
return None
def get_git_repo_url(repodir, remote='origin'):
import bb.process
# Try to get upstream repo location from origin remote
try:
stdout, _ = bb.process.run(['git', 'remote', '-v'], cwd=repodir)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
for line in stdout.splitlines():
splitline = line.split()
if len(splitline) > 1:
if splitline[0] == remote and scriptutils.is_src_url(splitline[1]):
return splitline[1]
return None
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare(config_only=False)
layers = OrderedDict()
for layerdir in tinfoil.config_data.getVar('BBLAYERS').split():
layerdata = OrderedDict()
layername = os.path.basename(layerdir)
logger.debug('layername %s, layerdir %s' % (layername, layerdir))
if layername in layers:
logger.warning('layername %s is not unique in configuration' % layername)
layername = os.path.basename(os.path.dirname(layerdir)) + '_' + os.path.basename(layerdir)
logger.debug('trying layername %s' % layername)
if layername in layers:
logger.error('Layer name %s is not unique in configuration' % layername)
sys.exit(2)
repodir = find_git_repo(layerdir)
if repodir:
remotebranch = get_git_remote_branch(repodir)
remote = 'origin'
if remotebranch and '/' in remotebranch:
rbsplit = remotebranch.split('/', 1)
layerdata['actual_branch'] = rbsplit[1]
remote = rbsplit[0]
layerdata['vcs_url'] = get_git_repo_url(repodir, remote)
if os.path.abspath(repodir) != os.path.abspath(layerdir):
layerdata['vcs_subdir'] = os.path.relpath(layerdir, repodir)
commit = get_git_head_commit(repodir)
if commit:
layerdata['vcs_commit'] = commit
layers[layername] = layerdata
json.dump(layers, args.output, indent=2)
def get_recipe(args): with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.logger.setLevel(logger.getEffectiveLevel()) tinfoil.prepare(config_only=True)
recipe = pkg2recipe(tinfoil, args.package)
print(' %s package provided by %s' % (args.package, recipe))
def pkg_dependencies(args): def get_recipe_info(tinfoil, recipe): try: info = tinfoil.get_recipe_info(recipe) except Exception: logger.error('Failed to get recipe info for: %s' % recipe) sys.exit(1) if not info: logger.warning('No recipe info found for: %s' % recipe) sys.exit(1) append_files = tinfoil.get_file_appends(info.fn) appends = True data = tinfoil.parse_recipe_file(info.fn, appends, append_files) data.pn = info.pn data.pv = info.pv return data
def find_dependencies(tinfoil, assume_provided, recipe_info, packages, rn, order):
spaces = ' ' * order
data = recipe_info[rn]
if args.native:
logger.debug('%s- %s' % (spaces, data.pn))
elif "-native" not in data.pn:
if "cross" not in data.pn:
logger.debug('%s- %s' % (spaces, data.pn))
depends = []
for dep in data.depends:
if dep not in assume_provided:
depends.append(dep)
# First find all dependencies not in package list.
for dep in depends:
if dep not in packages:
packages.append(dep)
dep_data = get_recipe_info(tinfoil, dep)
# Do this once now to reduce the number of bitbake calls.
dep_data.depends = dep_data.getVar('DEPENDS').split()
recipe_info[dep] = dep_data
# Then recursively analyze all of the dependencies for the current recipe.
for dep in depends:
find_dependencies(tinfoil, assume_provided, recipe_info, packages, dep, order + 1)
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare()
assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
logger.debug('assumed provided:')
for ap in sorted(assume_provided):
logger.debug(' - %s' % ap)
recipe = pkg2recipe(tinfoil, args.package)
data = get_recipe_info(tinfoil, recipe)
data.depends = []
depends = data.getVar('DEPENDS').split()
for dep in depends:
if dep not in assume_provided:
data.depends.append(dep)
recipe_info = dict([(recipe, data)])
packages = []
find_dependencies(tinfoil, assume_provided, recipe_info, packages, recipe, order=1)
print('\nThe following packages are required to build %s' % recipe)
for p in sorted(packages):
data = recipe_info[p]
if "-native" not in data.pn:
if "cross" not in data.pn:
print(" %s (%s)" % (data.pn,p))
if args.native:
print('\nThe following native packages are required to build %s' % recipe)
for p in sorted(packages):
data = recipe_info[p]
if "-native" in data.pn:
print(" %s(%s)" % (data.pn,p))
if "cross" in data.pn:
print(" %s(%s)" % (data.pn,p))
def default_config(): vlist = OrderedDict() vlist['PV'] = 'yes' vlist['SUMMARY'] = 'no' vlist['DESCRIPTION'] = 'no' vlist['SECTION'] = 'no' vlist['LICENSE'] = 'yes' vlist['HOMEPAGE'] = 'no' vlist['BUGTRACKER'] = 'no' vlist['PROVIDES'] = 'no' vlist['BBCLASSEXTEND'] = 'no' vlist['DEPENDS'] = 'no' vlist['PACKAGECONFIG'] = 'no' vlist['SRC_URI'] = 'yes' vlist['SRCREV'] = 'yes' vlist['EXTRA_OECONF'] = 'no' vlist['EXTRA_OESCONS'] = 'no' vlist['EXTRA_OECMAKE'] = 'no' vlist['EXTRA_OEMESON'] = 'no'
clist = OrderedDict()
clist['variables'] = vlist
clist['filepath'] = 'no'
clist['sha256sum'] = 'no'
clist['layerdir'] = 'no'
clist['layer'] = 'no'
clist['inherits'] = 'no'
clist['source_urls'] = 'no'
clist['packageconfig_opts'] = 'no'
clist['patches'] = 'no'
clist['packagedir'] = 'no'
return clist
def dump_config(args): config = default_config() f = open('default_config.json', 'w') json.dump(config, f, indent=2) logger.info('Default config list dumped to default_config.json')
def export_manifest_info(args):
def handle_value(value):
if value:
return oe.utils.squashspaces(value)
else:
return value
if args.config:
logger.debug('config: %s' % args.config)
f = open(args.config, 'r')
config = json.load(f, object_pairs_hook=OrderedDict)
else:
config = default_config()
if logger.isEnabledFor(logging.DEBUG):
print('Configuration:')
json.dump(config, sys.stdout, indent=2)
print('')
tmpoutdir = tempfile.mkdtemp(prefix=os.path.basename(__file__)+'-')
logger.debug('tmp dir: %s' % tmpoutdir)
# export manifest
shutil.copy2(args.manifest,os.path.join(tmpoutdir, "manifest"))
with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
tinfoil.logger.setLevel(logger.getEffectiveLevel())
tinfoil.prepare(config_only=False)
pkglist = get_pkg_list(args.manifest)
# export pkg list
f = open(os.path.join(tmpoutdir, "pkgs"), 'w')
for pkg in pkglist:
f.write('%s\n' % pkg)
f.close()
recipelist = []
for pkg in pkglist:
recipe = pkg2recipe(tinfoil,pkg)
if recipe:
if not recipe in recipelist:
recipelist.append(recipe)
recipelist.sort()
# export recipe list
f = open(os.path.join(tmpoutdir, "recipes"), 'w')
for recipe in recipelist:
f.write('%s\n' % recipe)
f.close()
try:
rvalues = OrderedDict()
for pn in sorted(recipelist):
logger.debug('Package: %s' % pn)
rd = tinfoil.parse_recipe(pn)
rvalues[pn] = OrderedDict()
for varname in config['variables']:
if config['variables'][varname] == 'yes':
rvalues[pn][varname] = handle_value(rd.getVar(varname))
fpth = rd.getVar('FILE')
layerdir = oe.recipeutils.find_layerdir(fpth)
if config['filepath'] == 'yes':
rvalues[pn]['filepath'] = os.path.relpath(fpth, layerdir)
if config['sha256sum'] == 'yes':
rvalues[pn]['sha256sum'] = bb.utils.sha256_file(fpth)
if config['layerdir'] == 'yes':
rvalues[pn]['layerdir'] = layerdir
if config['layer'] == 'yes':
rvalues[pn]['layer'] = os.path.basename(layerdir)
if config['inherits'] == 'yes':
gr = set(tinfoil.config_data.getVar("__inherit_cache") or [])
lr = set(rd.getVar("__inherit_cache") or [])
rvalues[pn]['inherits'] = sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr})
if config['source_urls'] == 'yes':
rvalues[pn]['source_urls'] = []
for url in (rd.getVar('SRC_URI') or '').split():
if not url.startswith('file://'):
url = url.split(';')[0]
rvalues[pn]['source_urls'].append(url)
if config['packageconfig_opts'] == 'yes':
rvalues[pn]['packageconfig_opts'] = OrderedDict()
for key in rd.getVarFlags('PACKAGECONFIG').keys():
if key == 'doc':
continue
rvalues[pn]['packageconfig_opts'][key] = rd.getVarFlag('PACKAGECONFIG', key, True)
if config['patches'] == 'yes':
patches = oe.recipeutils.get_recipe_patches(rd)
rvalues[pn]['patches'] = []
if patches:
recipeoutdir = os.path.join(tmpoutdir, pn, 'patches')
bb.utils.mkdirhier(recipeoutdir)
for patch in patches:
# Patches may be in other layers too
patchlayerdir = oe.recipeutils.find_layerdir(patch)
# patchlayerdir will be None for remote patches, which we ignore
# (since currently they are considered as part of sources)
if patchlayerdir:
rvalues[pn]['patches'].append((os.path.basename(patchlayerdir), os.path.relpath(patch, patchlayerdir)))
shutil.copy(patch, recipeoutdir)
if config['packagedir'] == 'yes':
pn_dir = os.path.join(tmpoutdir, pn)
bb.utils.mkdirhier(pn_dir)
f = open(os.path.join(pn_dir, 'recipe.json'), 'w')
json.dump(rvalues[pn], f, indent=2)
f.close()
with open(os.path.join(tmpoutdir, 'recipes.json'), 'w') as f:
json.dump(rvalues, f, indent=2)
if args.output:
outname = os.path.basename(args.output)
else:
outname = os.path.splitext(os.path.basename(args.manifest))[0]
if outname.endswith('.tar.gz'):
outname = outname[:-7]
elif outname.endswith('.tgz'):
outname = outname[:-4]
tarfn = outname
if tarfn.endswith(os.sep):
tarfn = tarfn[:-1]
if not tarfn.endswith(('.tar.gz', '.tgz')):
tarfn += '.tar.gz'
with open(tarfn, 'wb') as f:
with tarfile.open(None, "w:gz", f) as tar:
tar.add(tmpoutdir, outname)
finally:
shutil.rmtree(tmpoutdir)
def main(): parser = argparse_oe.ArgumentParser(description="Image manifest utility", epilog="Use %(prog)s --help to get help on a specific command") parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') subparsers = parser.add_subparsers(dest="subparser_name", title='subcommands', metavar='') subparsers.required = True
# get recipe info
parser_get_recipes = subparsers.add_parser('recipe-info',
help='Get recipe info',
description='Get recipe information for a package')
parser_get_recipes.add_argument('package', help='Package name')
parser_get_recipes.set_defaults(func=get_recipe)
# list runtime dependencies
parser_pkg_dep = subparsers.add_parser('list-depends',
help='List dependencies',
description='List dependencies required to build the package')
parser_pkg_dep.add_argument('--native', help='also print native and cross packages', action='store_true')
parser_pkg_dep.add_argument('package', help='Package name')
parser_pkg_dep.set_defaults(func=pkg_dependencies)
# list recipes
parser_recipes = subparsers.add_parser('list-recipes',
help='List recipes producing packages within an image',
description='Lists recipes producing the packages that went into an image, using the manifest and pkgdata')
parser_recipes.add_argument('manifest', help='Manifest file')
parser_recipes.set_defaults(func=list_recipes)
# list packages
parser_packages = subparsers.add_parser('list-packages',
help='List packages within an image',
description='Lists packages that went into an image, using the manifest')
parser_packages.add_argument('manifest', help='Manifest file')
parser_packages.set_defaults(func=list_packages)
# list layers
parser_layers = subparsers.add_parser('list-layers',
help='List included layers',
description='Lists included layers')
parser_layers.add_argument('-o', '--output', help='Output file - defaults to stdout if not specified',
default=sys.stdout, type=argparse.FileType('w'))
parser_layers.set_defaults(func=list_layers)
# dump default configuration file
parser_dconfig = subparsers.add_parser('dump-config',
help='Dump default config',
description='Dump default config to default_config.json')
parser_dconfig.set_defaults(func=dump_config)
# export recipe info for packages in manifest
parser_export = subparsers.add_parser('manifest-info',
help='Export recipe info for a manifest',
description='Export recipe information using the manifest')
parser_export.add_argument('-c', '--config', help='load config from json file')
parser_export.add_argument('-o', '--output', help='Output file (tarball) - defaults to manifest name if not specified')
parser_export.add_argument('manifest', help='Manifest file')
parser_export.set_defaults(func=export_manifest_info)
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
logger.debug("Debug Enabled")
elif args.quiet:
logger.setLevel(logging.ERROR)
ret = args.func(args)
return ret
if name == "main": try: ret = main() except Exception: ret = 1 import traceback traceback.print_exc() sys.exit(ret)