From 1a9f73d4a75a6c64fd90d10532b030a880c11353 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Tue, 7 May 2013 12:12:03 +0100 Subject: [PATCH] Split out recipe parsing and utility functions to a separate module To allow re-use outside of the update script, split out parsing setup code to a new recipeparse module. Also split out runcmd, get_layer, get_branch and logger_create functions to a separate utils module. Signed-off-by: Paul Eggleton --- layerindex/import.py | 42 ++-------- layerindex/recipeparse.py | 111 ++++++++++++++++++++++++++ layerindex/update.py | 160 ++++++-------------------------------- layerindex/utils.py | 65 ++++++++++++++++ 4 files changed, 203 insertions(+), 175 deletions(-) create mode 100644 layerindex/recipeparse.py create mode 100644 layerindex/utils.py diff --git a/layerindex/import.py b/layerindex/import.py index bb2060a..cb4a770 100755 --- a/layerindex/import.py +++ b/layerindex/import.py @@ -11,33 +11,10 @@ import sys import os.path import optparse -import logging import re +import utils -def logger_create(): - logger = logging.getLogger("LayerIndexImport") - loggerhandler = logging.StreamHandler() - loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) - logger.addHandler(loggerhandler) - logger.setLevel(logging.INFO) - return logger - -logger = logger_create() - - -def get_branch(branchname): - from layerindex.models import Branch - res = list(Branch.objects.filter(name=branchname)[:1]) - if res: - return res[0] - return None - -def get_layer(layername): - from layerindex.models import LayerItem - res = list(LayerItem.objects.filter(name=layername)[:1]) - if res: - return res[0] - return None +logger = utils.logger_create('LayerIndexImport') def main(): @@ -48,18 +25,9 @@ def main(): options, args = parser.parse_args(sys.argv) - # Get access to our Django model - newpath = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])) + '/..') - sys.path.append(newpath) - os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' - - from django.core.management import setup_environ - from django.conf import settings + utils.setup_django() from layerindex.models import LayerItem, LayerBranch, LayerDependency from django.db import transaction - import settings - - setup_environ(settings) import httplib conn = httplib.HTTPConnection("www.openembedded.org") @@ -72,7 +40,7 @@ def main(): nowiki_re = re.compile(r'') link_re = re.compile(r'\[(http.*) +link\]') readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') - master_branch = get_branch('master') + master_branch = utils.get_branch('master') core_layer = None transaction.enter_transaction_management() transaction.managed(True) @@ -163,7 +131,7 @@ def main(): layerbranch.save() if layer.name != 'openembedded-core': if not core_layer: - core_layer = get_layer('openembedded-core') + core_layer = utils.get_layer('openembedded-core') if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch diff --git a/layerindex/recipeparse.py b/layerindex/recipeparse.py new file mode 100644 index 0000000..2f9a668 --- /dev/null +++ b/layerindex/recipeparse.py @@ -0,0 +1,111 @@ +# Utility functions for parsing recipes using bitbake within layerindex-web +# +# Copyright (C) 2013 Intel Corporation +# Author: Paul Eggleton +# +# Licensed under the MIT license, see COPYING.MIT for details + +import sys +import os +import os.path +import utils +import tempfile + +class RecipeParseError(Exception): + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg + +def _setup_tinfoil(bitbakepath, enable_tracking): + sys.path.insert(0, bitbakepath + '/lib') + import bb.tinfoil + import bb.cooker + import bb.data + tinfoil = bb.tinfoil.Tinfoil() + if enable_tracking: + tinfoil.cooker.enableDataTracking() + tinfoil.prepare(config_only = True) + + return tinfoil + +def _parse_layer_conf(layerdir, data): + data.setVar('LAYERDIR', str(layerdir)) + if hasattr(bb, "cookerdata"): + # Newer BitBake + data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data) + else: + # Older BitBake (1.18 and below) + data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data) + data.expandVarref('LAYERDIR') + + +def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False): + if not nocheckout: + # Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc) + out = utils.runcmd("git checkout origin/%s" % branch.bitbake_branch, bitbakepath) + out = utils.runcmd("git clean -f -x", bitbakepath) + + # Skip sanity checks + os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS' + os.environ['DISABLE_SANITY_CHECKS'] = '1' + + fetchdir = settings.LAYER_FETCH_DIR + + # Ensure we have OE-Core set up to get some base configuration + core_layer = utils.get_layer(settings.CORE_LAYER_NAME) + if not core_layer: + raise RecipeParseError("Unable to find core layer %s in database; check CORE_LAYER_NAME setting" % settings.CORE_LAYER_NAME) + core_layerbranch = core_layer.get_layerbranch(branch.name) + core_branchname = branch.name + if core_layerbranch: + core_subdir = core_layerbranch.vcs_subdir + if core_layerbranch.actual_branch: + core_branchname = core_layerbranch.actual_branch + else: + core_subdir = 'meta' + core_urldir = core_layer.get_fetch_dir() + core_repodir = os.path.join(fetchdir, core_urldir) + core_layerdir = os.path.join(core_repodir, core_subdir) + if not nocheckout: + out = utils.runcmd("git checkout origin/%s" % core_branchname, core_repodir) + out = utils.runcmd("git clean -f -x", core_repodir) + # The directory above where this script exists should contain our conf/layer.conf, + # so add it to BBPATH along with the core layer directory + confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) + os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir)) + + # Change into a temporary directory so we don't write the cache and other files to the current dir + if not os.path.exists(settings.TEMP_BASE_DIR): + os.makedirs(settings.TEMP_BASE_DIR) + tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR) + os.chdir(tempdir) + + tinfoil = _setup_tinfoil(bitbakepath, enable_tracking) + + # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log) + oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True) + os.makedirs(oe_tmpdir) + + return (tinfoil, tempdir) + +def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch): + # Parse layer.conf files for this layer and its dependencies + # This is necessary not just because BBPATH needs to be set in order + # for include/require/inherit to work outside of the current directory + # or across layers, but also because custom variable values might be + # set in layer.conf. + config_data_copy = bb.data.createCopy(config_data) + _parse_layer_conf(layerdir, config_data_copy) + for dep in layerbranch.dependencies_set.all(): + depurldir = dep.dependency.get_fetch_dir() + deprepodir = os.path.join(fetchdir, depurldir) + deplayerbranch = dep.dependency.get_layerbranch(layerbranch.branch.name) + if not deplayerbranch: + raise RecipeParseError('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, layerbranch.branch.name)) + deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir) + _parse_layer_conf(deplayerdir, config_data_copy) + config_data_copy.delVar('LAYERDIR') + return config_data_copy + diff --git a/layerindex/update.py b/layerindex/update.py index f0cdbf1..ed67009 100755 --- a/layerindex/update.py +++ b/layerindex/update.py @@ -12,23 +12,16 @@ import sys import os.path import optparse import logging -import subprocess from datetime import datetime import fnmatch import re import tempfile import shutil from distutils.version import LooseVersion +import utils +import recipeparse -def logger_create(): - logger = logging.getLogger("LayerIndexUpdate") - loggerhandler = logging.StreamHandler() - loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) - logger.addHandler(loggerhandler) - logger.setLevel(logging.INFO) - return logger - -logger = logger_create() +logger = utils.logger_create('LayerIndexUpdate') # Ensure PythonGit is installed (buildhistory_analysis needs it) try: @@ -38,27 +31,6 @@ except ImportError: sys.exit(1) -def runcmd(cmd,destdir=None,printerr=True): - """ - execute command, raise CalledProcessError if fail - return output if succeed - """ - logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) - out = os.tmpfile() - try: - subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True) - except subprocess.CalledProcessError,e: - out.seek(0) - if printerr: - logger.error("%s" % out.read()) - raise e - - out.seek(0) - output = out.read() - logger.debug("output: %s" % output.rstrip() ) - return output - - machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$') bbclass_re = re.compile(r'classes/([^/.]*).bbclass$') def detect_file_type(path, subdir_start): @@ -150,30 +122,6 @@ def update_machine_conf_file(path, machine): break machine.description = desc -def parse_layer_conf(layerdir, data): - data.setVar('LAYERDIR', str(layerdir)) - if hasattr(bb, "cookerdata"): - # Newer BitBake - data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data) - else: - # Older BitBake (1.18 and below) - data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data) - data.expandVarref('LAYERDIR') - -def get_branch(branchname): - from layerindex.models import Branch - res = list(Branch.objects.filter(name=branchname)[:1]) - if res: - return res[0] - return None - -def get_layer(layername): - from layerindex.models import LayerItem - res = list(LayerItem.objects.filter(name=layername)[:1]) - if res: - return res[0] - return None - def main(): if LooseVersion(git.__version__) < '0.3.1': logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script") @@ -216,22 +164,14 @@ def main(): parser.print_help() sys.exit(1) - # Get access to our Django model - newpath = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])) + '/..') - sys.path.append(newpath) - os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' - - from django.core.management import setup_environ - from django.conf import settings + utils.setup_django() + import settings from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass from django.db import transaction - import settings - - setup_environ(settings) logger.setLevel(options.loglevel) - branch = get_branch(options.branch) + branch = utils.get_branch(options.branch) if not branch: logger.error("Specified branch %s is not valid" % options.branch) sys.exit(1) @@ -270,9 +210,9 @@ def main(): out = None try: if not os.path.exists(repodir): - out = runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir) + out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir) else: - out = runcmd("git fetch", repodir) + out = utils.runcmd("git fetch", repodir) except Exception as e: logger.error("Fetch of layer %s failed: %s" % (layer.name, str(e))) failedrepos.append(layer.vcs_url) @@ -285,54 +225,15 @@ def main(): logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL) if not os.path.exists(bitbakepath): - out = runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir) + out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir) else: - out = runcmd("git fetch", bitbakepath) + out = utils.runcmd("git fetch", bitbakepath) - if not options.nocheckout: - # Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc) - out = runcmd("git checkout origin/%s" % branch.bitbake_branch, bitbakepath) - out = runcmd("git clean -f -x", bitbakepath) - - # Skip sanity checks - os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS' - os.environ['DISABLE_SANITY_CHECKS'] = '1' - - # Ensure we have OE-Core set up to get some base configuration - core_layer = get_layer(settings.CORE_LAYER_NAME) - if not core_layer: - logger.error("Unable to find core layer %s in database; check CORE_LAYER_NAME setting" % settings.CORE_LAYER_NAME) + try: + (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout) + except recipeparse.RecipeParseError as e: + logger.error(str(e)) sys.exit(1) - core_layerbranch = core_layer.get_layerbranch(options.branch) - core_branchname = options.branch - if core_layerbranch: - core_subdir = core_layerbranch.vcs_subdir - if core_layerbranch.actual_branch: - core_branchname = core_layerbranch.actual_branch - else: - core_subdir = 'meta' - core_urldir = core_layer.get_fetch_dir() - core_repodir = os.path.join(fetchdir, core_urldir) - core_layerdir = os.path.join(core_repodir, core_subdir) - if not options.nocheckout: - out = runcmd("git checkout origin/%s" % core_branchname, core_repodir) - out = runcmd("git clean -f -x", core_repodir) - # The directory above where this script exists should contain our conf/layer.conf, - # so add it to BBPATH along with the core layer directory - confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) - os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir)) - - # Change into a temporary directory so we don't write the cache and other files to the current dir - if not os.path.exists(settings.TEMP_BASE_DIR): - os.makedirs(settings.TEMP_BASE_DIR) - tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR) - os.chdir(tempdir) - - sys.path.extend([bitbakepath + '/lib']) - import bb.tinfoil - import bb.cooker - tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(config_only = True) # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set tinfoil.config_data.setVar('SUMMARY', '') @@ -344,10 +245,6 @@ def main(): # why won't they just fix that?!) tinfoil.config_data.setVar('LICENSE', '') - # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log) - oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True) - os.makedirs(oe_tmpdir) - # Process and extract data from each layer for layer in layerquery: transaction.enter_transaction_management() @@ -419,8 +316,8 @@ def main(): if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload: # Check out appropriate branch if not options.nocheckout: - out = runcmd("git checkout origin/%s" % branchname, repodir) - out = runcmd("git clean -f -x", repodir) + out = utils.runcmd("git checkout origin/%s" % branchname, repodir) + out = utils.runcmd("git clean -f -x", repodir) if not os.path.exists(layerdir): if options.branch == 'master': @@ -439,25 +336,12 @@ def main(): logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc)) - # Parse layer.conf files for this layer and its dependencies - # This is necessary not just because BBPATH needs to be set in order - # for include/require/inherit to work outside of the current directory - # or across layers, but also because custom variable values might be - # set in layer.conf. - - config_data_copy = bb.data.createCopy(tinfoil.config_data) - parse_layer_conf(layerdir, config_data_copy) - for dep in layerbranch.dependencies_set.all(): - depurldir = dep.dependency.get_fetch_dir() - deprepodir = os.path.join(fetchdir, depurldir) - deplayerbranch = dep.dependency.get_layerbranch(options.branch) - if not deplayerbranch: - logger.error('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, options.branch)) - transaction.rollback() - continue - deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir) - parse_layer_conf(deplayerdir, config_data_copy) - config_data_copy.delVar('LAYERDIR') + try: + config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch) + except recipeparse.RecipeParseError as e: + logger.error(str(e)) + transaction.rollback() + continue if layerbranch.vcs_last_rev and not options.reload: try: diff --git a/layerindex/utils.py b/layerindex/utils.py new file mode 100644 index 0000000..32cdbb3 --- /dev/null +++ b/layerindex/utils.py @@ -0,0 +1,65 @@ +# Utilities for layerindex-web +# +# Copyright (C) 2013 Intel Corporation +# Author: Paul Eggleton +# +# Licensed under the MIT license, see COPYING.MIT for details + +import sys +import os.path +import subprocess +import logging + +def get_branch(branchname): + from layerindex.models import Branch + res = list(Branch.objects.filter(name=branchname)[:1]) + if res: + return res[0] + return None + +def get_layer(layername): + from layerindex.models import LayerItem + res = list(LayerItem.objects.filter(name=layername)[:1]) + if res: + return res[0] + return None + +def runcmd(cmd, destdir=None, printerr=True): + """ + execute command, raise CalledProcessError if fail + return output if succeed + """ + #logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir)) + out = os.tmpfile() + try: + subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True) + except subprocess.CalledProcessError,e: + out.seek(0) + if printerr: + logger.error("%s" % out.read()) + raise e + + out.seek(0) + output = out.read() + #logger.debug("output: %s" % output.rstrip() ) + return output + +def setup_django(): + # Get access to our Django model + newpath = os.path.abspath(os.path.dirname(__file__) + '/..') + sys.path.append(newpath) + os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' + + from django.core.management import setup_environ + from django.conf import settings + import settings + + setup_environ(settings) + +def logger_create(name): + logger = logging.getLogger(name) + loggerhandler = logging.StreamHandler() + loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(loggerhandler) + logger.setLevel(logging.INFO) + return logger