poky/scripts/crosstap

#!/usr/bin/env python3

Build a systemtap script for a given image, kernel

Effectively script extracts needed information from set of

'bitbake -e' commands and contructs proper invocation of stap on

host to build systemtap script for a given target.

By default script will compile scriptname.ko that could be copied

to taget and activated with 'staprun scriptname.ko' command. Or if

--remote user@hostname option is specified script will build, load

execute script on target.

This script is very similar and inspired by crosstap shell script.

The major difference that this script supports user-land related

systemtap script, whereas crosstap could deal only with scripts

related to kernel.

Copyright (c) 2018, Cisco Systems.

SPDX-License-Identifier: GPL-2.0-only

import sys import re import subprocess import os import optparse

class Stap(object): def init(self, script, module, remote): self.script = script self.module = module self.remote = remote self.stap = None self.sysroot = None self.runtime = None self.tapset = None self.arch = None self.cross_compile = None self.kernel_release = None self.target_path = None self.target_ld_library_path = None

    if not self.remote:
        if not self.module:
            # derive module name from script
            self.module = os.path.basename(self.script)
            if self.module[-4:] == ".stp":
                self.module = self.module[:-4]
                # replace - if any with _
                self.module = self.module.replace("-", "_")

def command(self, args):
    ret = []
    ret.append(self.stap)

    if self.remote:
        ret.append("--remote")
        ret.append(self.remote)
    else:
        ret.append("-p4")
        ret.append("-m")
        ret.append(self.module)

    ret.append("-a")
    ret.append(self.arch)

    ret.append("-B")
    ret.append("CROSS_COMPILE=" + self.cross_compile)

    ret.append("-r")
    ret.append(self.kernel_release)

    ret.append("-I")
    ret.append(self.tapset)

    ret.append("-R")
    ret.append(self.runtime)

    if self.sysroot:
        ret.append("--sysroot")
        ret.append(self.sysroot)

        ret.append("--sysenv=PATH=" + self.target_path)
        ret.append("--sysenv=LD_LIBRARY_PATH=" + self.target_ld_library_path)

    ret = ret + args

    ret.append(self.script)
    return ret

def additional_environment(self):
    ret = {}
    ret["SYSTEMTAP_DEBUGINFO_PATH"] = "+:.debug:build"
    return ret

def environment(self):
    ret = os.environ.copy()
    additional = self.additional_environment()
    for e in additional:
        ret[e] = additional[e]
    return ret

def display_command(self, args):
    additional_env = self.additional_environment()
    command = self.command(args)

    print("#!/bin/sh")
    for e in additional_env:
        print("export %s=\"%s\"" % (e, additional_env[e]))
    print(" ".join(command))

class BitbakeEnvInvocationException(Exception): def init(self, message): self.message = message

class BitbakeEnv(object): BITBAKE="bitbake"

def __init__(self, package):
    self.package = package
    self.cmd = BitbakeEnv.BITBAKE + " -e " + self.package
    self.popen = subprocess.Popen(self.cmd, shell=True,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)
    self.__lines = self.popen.stdout.readlines()
    self.popen.wait()

    self.lines = []
    for line in self.__lines:
            self.lines.append(line.decode('utf-8'))

def get_vars(self, vars):
    if self.popen.returncode:
        raise BitbakeEnvInvocationException(
            "\nFailed to execute '" + self.cmd +
            "' with the following message:\n" +
            ''.join(self.lines))

    search_patterns = []
    retdict = {}
    for var in vars:
        # regular not exported variable
        rexpr = "^" + var + "=\"(.*)\""
        re_compiled = re.compile(rexpr)
        search_patterns.append((var, re_compiled))

        # exported variable
        rexpr = "^export " + var + "=\"(.*)\""
        re_compiled = re.compile(rexpr)
        search_patterns.append((var, re_compiled))

        for line in self.lines:
            for var, rexpr in search_patterns:
                m = rexpr.match(line)
                if m:
                    value = m.group(1)
                    retdict[var] = value

    # fill variables values in order how they were requested
    ret = []
    for var in vars:
        ret.append(retdict.get(var))

    # if it is single value list return it as scalar, not the list
    if len(ret) == 1:
        ret = ret[0]

    return ret

class ParamDiscovery(object): SYMBOLS_CHECK_MESSAGE = """ WARNING: image '%s' does not have dbg-pkgs IMAGE_FEATURES enabled and no "image-combined-dbg" in inherited classes is specified. As result the image does not have symbols for user-land processes DWARF based probes. Consider adding 'dbg-pkgs' to EXTRA_IMAGE_FEATURES or adding "image-combined-dbg" to USER_CLASSES. I.e add this line 'USER_CLASSES += "image-combined-dbg"' to local.conf file.

Or you may use IMAGE_GEN_DEBUGFS="1" option, and then after build you need recombine/unpack image and image-dbg tarballs and pass resulting dir location with --sysroot option. """

def __init__(self, image):
    self.image = image

    self.image_rootfs = None
    self.image_features = None
    self.image_gen_debugfs = None
    self.inherit = None
    self.base_bindir = None
    self.base_sbindir = None
    self.base_libdir = None
    self.bindir = None
    self.sbindir = None
    self.libdir = None

    self.staging_bindir_toolchain = None
    self.target_prefix = None
    self.target_arch = None
    self.target_kernel_builddir = None

    self.staging_dir_native = None

    self.image_combined_dbg = False

def discover(self):
    if self.image:
        benv_image = BitbakeEnv(self.image)
        (self.image_rootfs,
         self.image_features,
         self.image_gen_debugfs,
         self.inherit,
         self.base_bindir,
         self.base_sbindir,
         self.base_libdir,
         self.bindir,
         self.sbindir,
         self.libdir
        ) = benv_image.get_vars(
             ("IMAGE_ROOTFS",
              "IMAGE_FEATURES",
              "IMAGE_GEN_DEBUGFS",
              "INHERIT",
              "base_bindir",
              "base_sbindir",
              "base_libdir",
              "bindir",
              "sbindir",
              "libdir"
              ))

    benv_kernel = BitbakeEnv("virtual/kernel")
    (self.staging_bindir_toolchain,
     self.target_prefix,
     self.target_arch,
     self.target_kernel_builddir
    ) = benv_kernel.get_vars(
         ("STAGING_BINDIR_TOOLCHAIN",
          "TARGET_PREFIX",
          "TRANSLATED_TARGET_ARCH",
          "B"
        ))

    benv_systemtap = BitbakeEnv("systemtap-native")
    (self.staging_dir_native
    ) = benv_systemtap.get_vars(["STAGING_DIR_NATIVE"])

    if self.inherit:
        if "image-combined-dbg" in self.inherit.split():
            self.image_combined_dbg = True

def check(self, sysroot_option):
    ret = True
    if self.image_rootfs:
        sysroot = self.image_rootfs
        if not os.path.isdir(self.image_rootfs):
            print("ERROR: Cannot find '" + sysroot +
                  "' directory. Was '" + self.image + "' image built?")
            ret = False

    stap = self.staging_dir_native + "/usr/bin/stap"
    if not os.path.isfile(stap):
        print("ERROR: Cannot find '" + stap +
              "'. Was 'systemtap-native' built?")
        ret = False

    if not os.path.isdir(self.target_kernel_builddir):
        print("ERROR: Cannot find '" + self.target_kernel_builddir +
              "' directory. Was 'kernel/virtual' built?")
        ret = False

    if not sysroot_option and self.image_rootfs:
        dbg_pkgs_found = False

        if self.image_features:
            image_features = self.image_features.split()
            if "dbg-pkgs" in image_features:
                dbg_pkgs_found = True

        if not dbg_pkgs_found \
           and not self.image_combined_dbg:
            print(ParamDiscovery.SYMBOLS_CHECK_MESSAGE % (self.image))

    if not ret:
        print("")

    return ret

def __map_systemtap_arch(self):
    a = self.target_arch
    ret = a
    if   re.match('(athlon|x86.64)$', a):
        ret = 'x86_64'
    elif re.match('i.86$', a):
        ret = 'i386'
    elif re.match('arm$', a):
        ret = 'arm'
    elif re.match('aarch64$', a):
        ret = 'arm64'
    elif re.match('mips(isa|)(32|64|)(r6|)(el|)$', a):
        ret = 'mips'
    elif re.match('p(pc|owerpc)(|64)', a):
        ret = 'powerpc'
    return ret

def fill_stap(self, stap):
    stap.stap = self.staging_dir_native + "/usr/bin/stap"
    if not stap.sysroot:
        if self.image_rootfs:
            if self.image_combined_dbg:
                stap.sysroot = self.image_rootfs + "-dbg"
            else:
                stap.sysroot = self.image_rootfs
    stap.runtime = self.staging_dir_native + "/usr/share/systemtap/runtime"
    stap.tapset = self.staging_dir_native + "/usr/share/systemtap/tapset"
    stap.arch = self.__map_systemtap_arch()
    stap.cross_compile = self.staging_bindir_toolchain + "/" + \
                         self.target_prefix
    stap.kernel_release = self.target_kernel_builddir

    # do we have standard that tells in which order these need to appear
    target_path = []
    if self.sbindir:
        target_path.append(self.sbindir)
    if self.bindir:
        target_path.append(self.bindir)
    if self.base_sbindir:
        target_path.append(self.base_sbindir)
    if self.base_bindir:
        target_path.append(self.base_bindir)
    stap.target_path = ":".join(target_path)

    target_ld_library_path = []
    if self.libdir:
        target_ld_library_path.append(self.libdir)
    if self.base_libdir:
        target_ld_library_path.append(self.base_libdir)
    stap.target_ld_library_path = ":".join(target_ld_library_path)

def main(): usage = """usage: %prog -s [options] [-- [systemtap options]]

%prog cross compile given SystemTap script against given image, kernel

It needs to run in environtment set for bitbake - it uses bitbake -e invocations to retrieve information to construct proper stap cross build invocation arguments. It assumes that systemtap-native is built in given bitbake workspace.

Anything after -- option is passed directly to stap.

Legacy script invocation style supported but deprecated: %prog user@hostname [systemtap options]

To enable most out of systemtap the following site.conf or local.conf configuration is recommended:

enables symbol + target binaries rootfs-dbg in workspace

IMAGE_GEN_DEBUGFS = "1" IMAGE_FSTYPES_DEBUGFS = "tar.bz2" USER_CLASSES += "image-combined-dbg"

enables kernel debug symbols

KERNEL_EXTRA_FEATURES:append = " features/debug/debug-kernel.scc"

minimal, just run-time systemtap configuration in target image

PACKAGECONFIG:pn-systemtap = "monitor"

add systemtap run-time into target image if it is not there yet

IMAGE_INSTALL:append = " systemtap" """ option_parser = optparse.OptionParser(usage=usage)

option_parser.add_option("-s", "--script", dest="script",
                         help="specify input script FILE name",
                         metavar="FILE")

option_parser.add_option("-i", "--image", dest="image",
                         help="specify image name for which script should be compiled")

option_parser.add_option("-r", "--remote", dest="remote",
                         help="specify username@hostname of remote target to run script "
                         "optional, it assumes that remote target can be accessed through ssh")

option_parser.add_option("-m", "--module", dest="module",
                         help="specify module name, optional, has effect only if --remote is not used, "
                         "if not specified module name will be derived from passed script name")

option_parser.add_option("-y", "--sysroot", dest="sysroot",
                         help="explicitely specify image sysroot location. May need to use it in case "
                         "when IMAGE_GEN_DEBUGFS=\"1\" option is used and recombined with symbols "
                         "in different location",
                         metavar="DIR")

option_parser.add_option("-o", "--out", dest="out",
                         action="store_true",
                         help="output shell script that equvivalent invocation of this script with "
                         "given set of arguments, in given bitbake environment. It could be stored in "
                         "separate shell script and could be repeated without incuring bitbake -e "
                         "invocation overhead",
                         default=False)

option_parser.add_option("-d", "--debug", dest="debug",
                         action="store_true",
                         help="enable debug output. Use this option to see resulting stap invocation",
                         default=False)

# is invocation follow syntax from orignal crosstap shell script
legacy_args = False

# check if we called the legacy way
if len(sys.argv) >= 3:
    if sys.argv[1].find("@") != -1 and os.path.exists(sys.argv[2]):
        legacy_args = True

        # fill options values for legacy invocation case
        options = optparse.Values
        options.script = sys.argv[2]
        options.remote = sys.argv[1]
        options.image = None
        options.module = None
        options.sysroot = None
        options.out = None
        options.debug = None
        remaining_args = sys.argv[3:]

if not legacy_args:
    (options, remaining_args) = option_parser.parse_args()

if not options.script or not os.path.exists(options.script):
    print("'-s FILE' option is missing\n")
    option_parser.print_help()
else:
    stap = Stap(options.script, options.module, options.remote)
    discovery = ParamDiscovery(options.image)
    discovery.discover()
    if not discovery.check(options.sysroot):
        option_parser.print_help()
    else:
        stap.sysroot = options.sysroot
        discovery.fill_stap(stap)

        if options.out:
            stap.display_command(remaining_args)
        else:
            cmd = stap.command(remaining_args)
            env = stap.environment()

            if options.debug:
                print(" ".join(cmd))

            os.execve(cmd[0], cmd, env)

main()