layerindex-web/lib/recipeparse.py

# Utility functions for parsing recipes using bitbake within layerindex-web
#
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details

import sys
import os
import os.path
import utils
import tempfile
import re
import fnmatch

class RecipeParseError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

def _setup_tinfoil(bitbakepath, enable_tracking):
    sys.path.insert(0, bitbakepath + '/lib')
    import bb.tinfoil
    import bb.cooker
    import bb.data
    try:
        tinfoil = bb.tinfoil.Tinfoil(tracking=enable_tracking)
    except TypeError:
        # old API
        tinfoil = bb.tinfoil.Tinfoil()
        if enable_tracking:
            tinfoil.cooker.enableDataTracking()
    tinfoil.prepare(config_only = True)

    return tinfoil

def _parse_layer_conf(layerdir, data):
    data.setVar('LAYERDIR', str(layerdir))
    if hasattr(bb, "cookerdata"):
        # Newer BitBake
        data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data)
    else:
        # Older BitBake (1.18 and below)
        data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data)
    data.expandVarref('LAYERDIR')


def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False, classic=False, logger=None):
    if not (nocheckout or classic):
        # Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc)
        if re.match('[0-9a-f]{40}', branch.bitbake_branch):
            # SHA1 hash
            bitbake_ref = branch.bitbake_branch
        else:
            # Branch name
            bitbake_ref = 'origin/%s' % branch.bitbake_branch
        out = utils.runcmd("git checkout %s" % bitbake_ref, bitbakepath, logger=logger)
        out = utils.runcmd("git clean -f -x", bitbakepath, logger=logger)

    # Skip sanity checks
    os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS'
    os.environ['DISABLE_SANITY_CHECKS'] = '1'

    fetchdir = settings.LAYER_FETCH_DIR

    if not classic:
        # Ensure we have OE-Core set up to get some base configuration
        core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
        if not core_layer:
            raise RecipeParseError("Unable to find core layer %s in database; create this layer or set the CORE_LAYER_NAME setting to point to the core layer" % settings.CORE_LAYER_NAME)
        core_layerbranch = core_layer.get_layerbranch(branch.name)
        core_branchname = branch.name
        if core_layerbranch:
            core_subdir = core_layerbranch.vcs_subdir
            if core_layerbranch.actual_branch:
                core_branchname = core_layerbranch.actual_branch
        else:
            core_subdir = 'meta'
        core_urldir = core_layer.get_fetch_dir()
        core_repodir = os.path.join(fetchdir, core_urldir)
        core_layerdir = os.path.join(core_repodir, core_subdir)
        if not nocheckout:
            out = utils.runcmd("git checkout origin/%s" % core_branchname, core_repodir, logger=logger)
            out = utils.runcmd("git clean -f -x", core_repodir, logger=logger)
        if not os.path.exists(os.path.join(core_layerdir, 'conf/bitbake.conf')):
            raise RecipeParseError("conf/bitbake.conf not found in core layer %s - is subdirectory set correctly?" % core_layer.name)
        # The directory above where this script exists should contain our conf/layer.conf,
        # so add it to BBPATH along with the core layer directory
        confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
        os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir))

    # Change into a temporary directory so we don't write the cache and other files to the current dir
    if not os.path.exists(settings.TEMP_BASE_DIR):
        os.makedirs(settings.TEMP_BASE_DIR)
    tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
    os.chdir(tempdir)

    tinfoil = _setup_tinfoil(bitbakepath, enable_tracking)

    # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log)
    oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True)
    if not os.path.exists(oe_tmpdir):
        os.makedirs(oe_tmpdir)

    return (tinfoil, tempdir)

def checkout_layer_branch(layerbranch, repodir, logger=None):
    if layerbranch.actual_branch:
        branchname = layerbranch.actual_branch
    else:
        branchname = layerbranch.branch.name
    out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
    out = utils.runcmd("git clean -f -x", repodir, logger=logger)

def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch):
    # Parse layer.conf files for this layer and its dependencies
    # This is necessary not just because BBPATH needs to be set in order
    # for include/require/inherit to work outside of the current directory
    # or across layers, but also because custom variable values might be
    # set in layer.conf.
    config_data_copy = bb.data.createCopy(config_data)
    _parse_layer_conf(layerdir, config_data_copy)
    for dep in layerbranch.dependencies_set.all():
        depurldir = dep.dependency.get_fetch_dir()
        deprepodir = os.path.join(fetchdir, depurldir)
        deplayerbranch = dep.dependency.get_layerbranch(layerbranch.branch.name)
        if not deplayerbranch:
            raise RecipeParseError('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, layerbranch.branch.name))
        deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir)
        _parse_layer_conf(deplayerdir, config_data_copy)
    config_data_copy.delVar('LAYERDIR')
    return config_data_copy

def get_var_files(fn, varlist, d):
    import bb.cache
    varfiles = {}
    envdata = bb.cache.Cache.loadDataFull(fn, [], d)
    for v in varlist:
        history = envdata.varhistory.get_variable_files(v)
        if history:
            actualfile = history[-1]
        else:
            actualfile = None
        varfiles[v] = actualfile

    return varfiles

machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
bbclass_re = re.compile(r'classes/([^/.]*).bbclass$')
def detect_file_type(path, subdir_start):
    typename = None
    if fnmatch.fnmatch(path, "*.bb"):
        typename = 'recipe'
    elif fnmatch.fnmatch(path, "*.bbappend"):
        typename = 'bbappend'
    else:
        # Check if it's a machine conf file
        subpath = path[len(subdir_start):]
        res = machine_conf_re.match(subpath)
        if res:
            typename = 'machine'
            return (typename, None, res.group(1))
        else:
            res = bbclass_re.match(subpath)
            if res:
                typename = 'bbclass'
                return (typename, None, res.group(1))

    if typename == 'recipe' or typename == 'bbappend':
        if subdir_start:
            filepath = os.path.relpath(os.path.dirname(path), subdir_start)
        else:
            filepath = os.path.dirname(path)
        return (typename, filepath, os.path.basename(path))

    return (None, None, None)

def split_recipe_fn(path):
    splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
    pn = splitfn[0]
    if len(splitfn) > 1:
        pv = splitfn[1]
    else:
        pv = "1.0"
    return (pn, pv)