MIRRORS/poky
1
0
Fork 0
mirror of git://git.yoctoproject.org/poky.git synced 2025-07-19 21:09:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity
b4932a18ab
poky/scripts/oe-pkgdata-util
Richard Purdie ffae400179 meta/lib+scripts: Convert to SPDX license headers 
This adds SPDX license headers in place of the wide assortment of things
currently in our script headers. We default to GPL-2.0-only except for the
oeqa code where it was clearly submitted and marked as MIT on the most part
or some scripts which had the "or later" GPL versioning.

The patch also drops other obsolete bits of file headers where they were
encoountered such as editor modelines, obsolete maintainer information or
the phrase "All rights reserved" which is now obsolete and not required in
copyright headers (in this case its actually confusing for licensing as all
rights were not reserved).

More work is needed for OE-Core but this takes care of the bulk of the scripts
and meta/lib directories.

The top level LICENSE files are tweaked to match the new structure and the
SPDX naming.

(From OE-Core rev: f8c9c511b5f1b7dbd45b77f345cb6c048ae6763e)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2019-05-09 16:31:55 +01:00

623 lines
27 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# OpenEmbedded pkgdata utility
#
# Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Copyright 2012-2015 Intel Corporation
#
# SPDX-License-Identifier: GPL-2.0-only
#
import sys
import os
import os.path
import fnmatch
import re
import argparse
import logging
from collections import defaultdict, OrderedDict
scripts_path = os.path.dirname(os.path.realpath(__file__))
lib_path = scripts_path + '/lib'
sys.path = sys.path + [lib_path]
import scriptutils
import argparse_oe
logger = scriptutils.logger_create('pkgdatautil')
def tinfoil_init():
import bb.tinfoil
import logging
tinfoil = bb.tinfoil.Tinfoil()
tinfoil.logger.setLevel(logging.WARNING)
tinfoil.prepare(True)
return tinfoil
def glob(args):
# Handle both multiple arguments and multiple values within an arg (old syntax)
globs = []
for globitem in args.glob:
globs.extend(globitem.split())
if not os.path.exists(args.pkglistfile):
logger.error('Unable to find package list file %s' % args.pkglistfile)
sys.exit(1)
skipval = "-locale-|^locale-base-|-dev$|-doc$|-dbg$|-staticdev$|^kernel-module-"
if args.exclude:
skipval += "|" + args.exclude
skipregex = re.compile(skipval)
skippedpkgs = set()
mappedpkgs = set()
with open(args.pkglistfile, 'r') as f:
for line in f:
fields = line.rstrip().split()
if not fields:
continue
pkg = fields[0]
# We don't care about other args (used to need the package architecture but the
# new pkgdata structure avoids the need for that)
# Skip packages for which there is no point applying globs
if skipregex.search(pkg):
logger.debug("%s -> !!" % pkg)
skippedpkgs.add(pkg)
continue
# Skip packages that already match the globs, so if e.g. a dev package
# is already installed and thus in the list, we don't process it any further
# Most of these will be caught by skipregex already, but just in case...
already = False
for g in globs:
if fnmatch.fnmatchcase(pkg, g):
already = True
break
if already:
skippedpkgs.add(pkg)
logger.debug("%s -> !" % pkg)
continue
# Define some functions
def revpkgdata(pkgn):
return os.path.join(args.pkgdata_dir, "runtime-reverse", pkgn)
def fwdpkgdata(pkgn):
return os.path.join(args.pkgdata_dir, "runtime", pkgn)
def readpn(pkgdata_file):
pn = ""
with open(pkgdata_file, 'r') as f:
for line in f:
if line.startswith("PN:"):
pn = line.split(': ')[1].rstrip()
return pn
def readrenamed(pkgdata_file):
renamed = ""
pn = os.path.basename(pkgdata_file)
with open(pkgdata_file, 'r') as f:
for line in f:
if line.startswith("PKG_%s:" % pn):
renamed = line.split(': ')[1].rstrip()
return renamed
# Main processing loop
for g in globs:
mappedpkg = ""
# First just try substitution (i.e. packagename -> packagename-dev)
newpkg = g.replace("*", pkg)
revlink = revpkgdata(newpkg)
if os.path.exists(revlink):
mappedpkg = os.path.basename(os.readlink(revlink))
fwdfile = fwdpkgdata(mappedpkg)
if os.path.exists(fwdfile):
mappedpkg = readrenamed(fwdfile)
if not os.path.exists(fwdfile + ".packaged"):
mappedpkg = ""
else:
revlink = revpkgdata(pkg)
if os.path.exists(revlink):
# Check if we can map after undoing the package renaming (by resolving the symlink)
origpkg = os.path.basename(os.readlink(revlink))
newpkg = g.replace("*", origpkg)
fwdfile = fwdpkgdata(newpkg)
if os.path.exists(fwdfile):
mappedpkg = readrenamed(fwdfile)
else:
# That didn't work, so now get the PN, substitute that, then map in the other direction
pn = readpn(revlink)
newpkg = g.replace("*", pn)
fwdfile = fwdpkgdata(newpkg)
if os.path.exists(fwdfile):
mappedpkg = readrenamed(fwdfile)
if not os.path.exists(fwdfile + ".packaged"):
mappedpkg = ""
else:
# Package doesn't even exist...
logger.debug("%s is not a valid package!" % (pkg))
break
if mappedpkg:
logger.debug("%s (%s) -> %s" % (pkg, g, mappedpkg))
mappedpkgs.add(mappedpkg)
else:
logger.debug("%s (%s) -> ?" % (pkg, g))
logger.debug("------")
print("\n".join(mappedpkgs - skippedpkgs))
def read_value(args):
# Handle both multiple arguments and multiple values within an arg (old syntax)
packages = []
if args.file:
with open(args.file, 'r') as f:
for line in f:
splitline = line.split()
if splitline:
packages.append(splitline[0])
else:
for pkgitem in args.pkg:
packages.extend(pkgitem.split())
if not packages:
logger.error("No packages specified")
sys.exit(1)
def readvar(pkgdata_file, valuename, mappedpkg):
val = ""
with open(pkgdata_file, 'r') as f:
for line in f:
if (line.startswith(valuename + ":") or
line.startswith(valuename + "_" + mappedpkg + ":")):
val = line.split(': ', 1)[1].rstrip()
return val
logger.debug("read-value('%s', '%s' '%s')" % (args.pkgdata_dir, args.valuename, packages))
for package in packages:
pkg_split = package.split('_')
pkg_name = pkg_split[0]
logger.debug("package: '%s'" % pkg_name)
revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg_name)
logger.debug(revlink)
if os.path.exists(revlink):
mappedpkg = os.path.basename(os.readlink(revlink))
qvar = args.valuename
value = readvar(revlink, qvar, mappedpkg)
if qvar == "PKGSIZE":
# PKGSIZE is now in bytes, but we we want it in KB
pkgsize = (int(value) + 1024 // 2) // 1024
value = "%d" % pkgsize
if args.unescape:
import codecs
# escape_decode() unescapes backslash encodings in byte streams
value = codecs.escape_decode(bytes(value, "utf-8"))[0].decode("utf-8")
if args.prefix_name:
print('%s %s' % (pkg_name, value))
else:
print(value)
else:
logger.debug("revlink %s does not exist", revlink)
def lookup_pkglist(pkgs, pkgdata_dir, reverse):
if reverse:
mappings = OrderedDict()
for pkg in pkgs:
revlink = os.path.join(pkgdata_dir, "runtime-reverse", pkg)
logger.debug(revlink)
if os.path.exists(revlink):
mappings[pkg] = os.path.basename(os.readlink(revlink))
else:
mappings = defaultdict(list)
for pkg in pkgs:
pkgfile = os.path.join(pkgdata_dir, 'runtime', pkg)
if os.path.exists(pkgfile):
with open(pkgfile, 'r') as f:
for line in f:
fields = line.rstrip().split(': ')
if fields[0] == 'PKG_%s' % pkg:
mappings[pkg].append(fields[1])
break
return mappings
def lookup_pkg(args):
# Handle both multiple arguments and multiple values within an arg (old syntax)
pkgs = []
for pkgitem in args.pkg:
pkgs.extend(pkgitem.split())
mappings = lookup_pkglist(pkgs, args.pkgdata_dir, args.reverse)
if len(mappings) < len(pkgs):
missing = list(set(pkgs) - set(mappings.keys()))
logger.error("The following packages could not be found: %s" % ', '.join(missing))
sys.exit(1)
if args.reverse:
items = list(mappings.values())
else:
items = []
for pkg in pkgs:
items.extend(mappings.get(pkg, []))
print('\n'.join(items))
def lookup_recipe(args):
def parse_pkgdatafile(pkgdatafile):
with open(pkgdatafile, 'r') as f:
found = False
for line in f:
if line.startswith('PN:'):
print("%s" % line.split(':', 1)[1].strip())
found = True
break
if not found:
logger.error("Unable to find PN entry in %s" % pkgdatafile)
sys.exit(1)
# Handle both multiple arguments and multiple values within an arg (old syntax)
pkgs = []
for pkgitem in args.pkg:
pkgs.extend(pkgitem.split())
for pkg in pkgs:
providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg)
if os.path.exists(providepkgpath):
for f in os.listdir(providepkgpath):
if f != pkg:
print("%s is in the RPROVIDES of %s:" % (pkg, f))
pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
parse_pkgdatafile(pkgdatafile)
continue
pkgdatafile = os.path.join(args.pkgdata_dir, 'runtime-reverse', pkg)
if os.path.exists(pkgdatafile):
parse_pkgdatafile(pkgdatafile)
else:
if args.carryon:
print("The following packages could not be found: %s" % pkg)
else:
logger.error("The following packages could not be found: %s" % pkg)
sys.exit(1)
def package_info(args):
def parse_pkgdatafile(pkgdatafile):
vars = ['PKGV', 'PKGE', 'PKGR', 'PN', 'PV', 'PE', 'PR', 'PKGSIZE']
if args.extra:
vars += args.extra
with open(pkgdatafile, 'r') as f:
vals = dict()
extra = ''
for line in f:
for var in vars:
m = re.match(var + '(?:_\S+)?:\s*(.+?)\s*$', line)
if m:
vals[var] = m.group(1)
pkg_version = vals['PKGV'] or ''
recipe = vals['PN'] or ''
recipe_version = vals['PV'] or ''
pkg_size = vals['PKGSIZE'] or ''
if 'PKGE' in vals:
pkg_version = vals['PKGE'] + ":" + pkg_version
if 'PKGR' in vals:
pkg_version = pkg_version + "-" + vals['PKGR']
if 'PE' in vals:
recipe_version = vals['PE'] + ":" + recipe_version
if 'PR' in vals:
recipe_version = recipe_version + "-" + vals['PR']
if args.extra:
for var in args.extra:
if var in vals:
val = re.sub(r'\s+', ' ', vals[var])
extra += ' "%s"' % val
print("%s %s %s %s %s%s" % (pkg, pkg_version, recipe, recipe_version, pkg_size, extra))
# Handle both multiple arguments and multiple values within an arg (old syntax)
packages = []
if args.file:
with open(args.file, 'r') as f:
for line in f:
splitline = line.split()
if splitline:
packages.append(splitline[0])
else:
for pkgitem in args.pkg:
packages.extend(pkgitem.split())
if not packages:
logger.error("No packages specified")
sys.exit(1)
for pkg in packages:
providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg)
if os.path.exists(providepkgpath):
for f in os.listdir(providepkgpath):
if f != pkg:
print("%s is in the RPROVIDES of %s:" % (pkg, f))
pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
parse_pkgdatafile(pkgdatafile)
continue
pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
if not os.path.exists(pkgdatafile):
logger.error("Unable to find any built runtime package named %s" % pkg)
sys.exit(1)
parse_pkgdatafile(pkgdatafile)
def get_recipe_pkgs(pkgdata_dir, recipe, unpackaged):
recipedatafile = os.path.join(pkgdata_dir, recipe)
if not os.path.exists(recipedatafile):
logger.error("Unable to find packaged recipe with name %s" % recipe)
sys.exit(1)
packages = []
with open(recipedatafile, 'r') as f:
for line in f:
fields = line.rstrip().split(': ')
if fields[0] == 'PACKAGES':
packages = fields[1].split()
break
if not unpackaged:
pkglist = []
for pkg in packages:
if os.path.exists(os.path.join(pkgdata_dir, 'runtime', '%s.packaged' % pkg)):
pkglist.append(pkg)
return pkglist
else:
return packages
def list_pkgs(args):
found = False
def matchpkg(pkg):
if args.pkgspec:
matched = False
for pkgspec in args.pkgspec:
if fnmatch.fnmatchcase(pkg, pkgspec):
matched = True
break
if not matched:
return False
if not args.unpackaged:
if args.runtime:
revlink = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
if os.path.exists(revlink):
# We're unlikely to get here if the package was not packaged, but just in case
# we add the symlinks for unpackaged files in the future
mappedpkg = os.path.basename(os.readlink(revlink))
if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % mappedpkg)):
return False
else:
return False
else:
if not os.path.exists(os.path.join(args.pkgdata_dir, 'runtime', '%s.packaged' % pkg)):
return False
return True
if args.recipe:
packages = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged)
if args.runtime:
pkglist = []
runtime_pkgs = lookup_pkglist(packages, args.pkgdata_dir, False)
for rtpkgs in runtime_pkgs.values():
pkglist.extend(rtpkgs)
else:
pkglist = packages
for pkg in pkglist:
if matchpkg(pkg):
found = True
print("%s" % pkg)
else:
if args.runtime:
searchdir = 'runtime-reverse'
else:
searchdir = 'runtime'
for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, searchdir)):
for fn in files:
if fn.endswith('.packaged'):
continue
if matchpkg(fn):
found = True
print("%s" % fn)
if not found:
if args.pkgspec:
logger.error("Unable to find any package matching %s" % args.pkgspec)
else:
logger.error("No packages found")
sys.exit(1)
def list_pkg_files(args):
import json
def parse_pkgdatafile(pkgdatafile, long=False):
with open(pkgdatafile, 'r') as f:
found = False
for line in f:
if line.startswith('FILES_INFO:'):
found = True
val = line.split(':', 1)[1].strip()
dictval = json.loads(val)
if long:
width = max(map(len, dictval), default=0)
for fullpth in sorted(dictval):
print("\t{:{width}}\t{}".format(fullpth, dictval[fullpth], width=width))
else:
for fullpth in sorted(dictval):
print("\t%s" % fullpth)
break
if not found:
logger.error("Unable to find FILES_INFO entry in %s" % pkgdatafile)
sys.exit(1)
if args.recipe:
if args.pkg:
logger.error("list-pkg-files: If -p/--recipe is specified then a package name cannot be specified")
sys.exit(1)
recipepkglist = get_recipe_pkgs(args.pkgdata_dir, args.recipe, args.unpackaged)
if args.runtime:
pkglist = []
runtime_pkgs = lookup_pkglist(recipepkglist, args.pkgdata_dir, False)
for rtpkgs in runtime_pkgs.values():
pkglist.extend(rtpkgs)
else:
pkglist = recipepkglist
else:
if not args.pkg:
logger.error("list-pkg-files: If -p/--recipe is not specified then at least one package name must be specified")
sys.exit(1)
pkglist = args.pkg
for pkg in sorted(pkglist):
print("%s:" % pkg)
if args.runtime:
pkgdatafile = os.path.join(args.pkgdata_dir, "runtime-reverse", pkg)
if not os.path.exists(pkgdatafile):
if args.recipe:
# This package was empty and thus never packaged, ignore
continue
logger.error("Unable to find any built runtime package named %s" % pkg)
sys.exit(1)
parse_pkgdatafile(pkgdatafile, args.long)
else:
providepkgpath = os.path.join(args.pkgdata_dir, "runtime-rprovides", pkg)
if os.path.exists(providepkgpath):
for f in os.listdir(providepkgpath):
if f != pkg:
print("%s is in the RPROVIDES of %s:" % (pkg, f))
pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", f)
parse_pkgdatafile(pkgdatafile, args.long)
continue
pkgdatafile = os.path.join(args.pkgdata_dir, "runtime", pkg)
if not os.path.exists(pkgdatafile):
logger.error("Unable to find any built recipe-space package named %s" % pkg)
sys.exit(1)
parse_pkgdatafile(pkgdatafile, args.long)
def find_path(args):
import json
found = False
for root, dirs, files in os.walk(os.path.join(args.pkgdata_dir, 'runtime')):
for fn in files:
with open(os.path.join(root,fn)) as f:
for line in f:
if line.startswith('FILES_INFO:'):
val = line.split(':', 1)[1].strip()
dictval = json.loads(val)
for fullpth in dictval.keys():
if fnmatch.fnmatchcase(fullpth, args.targetpath):
found = True
print("%s: %s" % (fn, fullpth))
break
if not found:
logger.error("Unable to find any package producing path %s" % args.targetpath)
sys.exit(1)
def main():
parser = argparse_oe.ArgumentParser(description="OpenEmbedded pkgdata tool - queries the pkgdata files written out during do_package",
epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
parser.add_argument('-p', '--pkgdata-dir', help='Path to pkgdata directory (determined automatically if not specified)')
subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')
subparsers.required = True
parser_lookup_pkg = subparsers.add_parser('lookup-pkg',
help='Translate between recipe-space package names and runtime package names',
description='Looks up the specified recipe-space package name(s) to see what the final runtime package name is (e.g. glibc becomes libc6), or with -r/--reverse looks up the other way.')
parser_lookup_pkg.add_argument('pkg', nargs='+', help='Package name to look up')
parser_lookup_pkg.add_argument('-r', '--reverse', help='Switch to looking up recipe-space package names from runtime package names', action='store_true')
parser_lookup_pkg.set_defaults(func=lookup_pkg)
parser_list_pkgs = subparsers.add_parser('list-pkgs',
help='List packages',
description='Lists packages that have been built')
parser_list_pkgs.add_argument('pkgspec', nargs='*', help='Package name to search for (wildcards * ? allowed, use quotes to avoid shell expansion)')
parser_list_pkgs.add_argument('-r', '--runtime', help='Show runtime package names instead of recipe-space package names', action='store_true')
parser_list_pkgs.add_argument('-p', '--recipe', help='Limit to packages produced by the specified recipe')
parser_list_pkgs.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages', action='store_true')
parser_list_pkgs.set_defaults(func=list_pkgs)
parser_list_pkg_files = subparsers.add_parser('list-pkg-files',
help='List files within a package',
description='Lists files included in one or more packages')
parser_list_pkg_files.add_argument('pkg', nargs='*', help='Package name to report on (if -p/--recipe is not specified)')
parser_list_pkg_files.add_argument('-r', '--runtime', help='Specified package(s) are runtime package names instead of recipe-space package names', action='store_true')
parser_list_pkg_files.add_argument('-p', '--recipe', help='Report on all packages produced by the specified recipe')
parser_list_pkg_files.add_argument('-u', '--unpackaged', help='Include unpackaged (i.e. empty) packages (only useful with -p/--recipe)', action='store_true')
parser_list_pkg_files.add_argument('-l', '--long', help='Show more information per file', action='store_true')
parser_list_pkg_files.set_defaults(func=list_pkg_files)
parser_lookup_recipe = subparsers.add_parser('lookup-recipe',
help='Find recipe producing one or more packages',
description='Looks up the specified runtime package(s) to see which recipe they were produced by')
parser_lookup_recipe.add_argument('pkg', nargs='+', help='Runtime package name to look up')
parser_lookup_recipe.add_argument('-c', '--continue', dest="carryon", help='Continue looking up recipes even if we can not find one', action='store_true')
parser_lookup_recipe.set_defaults(func=lookup_recipe)
parser_package_info = subparsers.add_parser('package-info',
help='Show version, recipe and size information for one or more packages',
description='Looks up the specified runtime package(s) and display information')
parser_package_info.add_argument('pkg', nargs='*', help='Runtime package name to look up')
parser_package_info.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)')
parser_package_info.add_argument('-e', '--extra', help='Extra variables to display, e.g., LICENSE (can be specified multiple times)', action='append')
parser_package_info.set_defaults(func=package_info)
parser_find_path = subparsers.add_parser('find-path',
help='Find package providing a target path',
description='Finds the recipe-space package providing the specified target path')
parser_find_path.add_argument('targetpath', help='Path to find (wildcards * ? allowed, use quotes to avoid shell expansion)')
parser_find_path.set_defaults(func=find_path)
parser_read_value = subparsers.add_parser('read-value',
help='Read any pkgdata value for one or more packages',
description='Reads the named value from the pkgdata files for the specified packages')
parser_read_value.add_argument('valuename', help='Name of the value to look up')
parser_read_value.add_argument('pkg', nargs='*', help='Runtime package name to look up')
parser_read_value.add_argument('-f', '--file', help='Read package names from the specified file (one per line, first field only)')
parser_read_value.add_argument('-n', '--prefix-name', help='Prefix output with package name', action='store_true')
parser_read_value.add_argument('-u', '--unescape', help='Expand escapes such as \\n', action='store_true')
parser_read_value.set_defaults(func=read_value)
parser_glob = subparsers.add_parser('glob',
help='Expand package name glob expression',
description='Expands one or more glob expressions over the packages listed in pkglistfile')
parser_glob.add_argument('pkglistfile', help='File listing packages (one package name per line)')
parser_glob.add_argument('glob', nargs="+", help='Glob expression for package names, e.g. *-dev')
parser_glob.add_argument('-x', '--exclude', help='Exclude packages matching specified regex from the glob operation')
parser_glob.set_defaults(func=glob)
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
if not args.pkgdata_dir:
import scriptpath
bitbakepath = scriptpath.add_bitbake_lib_path()
if not bitbakepath:
logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
sys.exit(1)
logger.debug('Found bitbake path: %s' % bitbakepath)
tinfoil = tinfoil_init()
try:
args.pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
finally:
tinfoil.shutdown()
logger.debug('Value of PKGDATA_DIR is "%s"' % args.pkgdata_dir)
if not args.pkgdata_dir:
logger.error('Unable to determine pkgdata directory from PKGDATA_DIR')
sys.exit(1)
if not os.path.exists(args.pkgdata_dir):
logger.error('Unable to find pkgdata directory %s' % args.pkgdata_dir)
sys.exit(1)
ret = args.func(args)
return ret
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink