mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 20:59:01 +02:00
Split out recipe parsing and utility functions to a separate module
To allow re-use outside of the update script, split out parsing setup code to a new recipeparse module. Also split out runcmd, get_layer, get_branch and logger_create functions to a separate utils module. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton
parent
8be1adddb6
commit
1a9f73d4a7
|
|
@ -11,33 +11,10 @@
|
|||
import sys
|
||||
import os.path
|
||||
import optparse
|
||||
import logging
|
||||
import re
|
||||
import utils
|
||||
|
||||
def logger_create():
|
||||
logger = logging.getLogger("LayerIndexImport")
|
||||
loggerhandler = logging.StreamHandler()
|
||||
loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
||||
logger.addHandler(loggerhandler)
|
||||
logger.setLevel(logging.INFO)
|
||||
return logger
|
||||
|
||||
logger = logger_create()
|
||||
|
||||
|
||||
def get_branch(branchname):
|
||||
from layerindex.models import Branch
|
||||
res = list(Branch.objects.filter(name=branchname)[:1])
|
||||
if res:
|
||||
return res[0]
|
||||
return None
|
||||
|
||||
def get_layer(layername):
|
||||
from layerindex.models import LayerItem
|
||||
res = list(LayerItem.objects.filter(name=layername)[:1])
|
||||
if res:
|
||||
return res[0]
|
||||
return None
|
||||
logger = utils.logger_create('LayerIndexImport')
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -48,18 +25,9 @@ def main():
|
|||
|
||||
options, args = parser.parse_args(sys.argv)
|
||||
|
||||
# Get access to our Django model
|
||||
newpath = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])) + '/..')
|
||||
sys.path.append(newpath)
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
|
||||
|
||||
from django.core.management import setup_environ
|
||||
from django.conf import settings
|
||||
utils.setup_django()
|
||||
from layerindex.models import LayerItem, LayerBranch, LayerDependency
|
||||
from django.db import transaction
|
||||
import settings
|
||||
|
||||
setup_environ(settings)
|
||||
|
||||
import httplib
|
||||
conn = httplib.HTTPConnection("www.openembedded.org")
|
||||
|
|
@ -72,7 +40,7 @@ def main():
|
|||
nowiki_re = re.compile(r'</?nowiki>')
|
||||
link_re = re.compile(r'\[(http.*) +link\]')
|
||||
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
|
||||
master_branch = get_branch('master')
|
||||
master_branch = utils.get_branch('master')
|
||||
core_layer = None
|
||||
transaction.enter_transaction_management()
|
||||
transaction.managed(True)
|
||||
|
|
@ -163,7 +131,7 @@ def main():
|
|||
layerbranch.save()
|
||||
if layer.name != 'openembedded-core':
|
||||
if not core_layer:
|
||||
core_layer = get_layer('openembedded-core')
|
||||
core_layer = utils.get_layer('openembedded-core')
|
||||
if core_layer:
|
||||
layerdep = LayerDependency()
|
||||
layerdep.layerbranch = layerbranch
|
||||
|
|
|
|||
111
layerindex/recipeparse.py
Normal file
111
layerindex/recipeparse.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
# Utility functions for parsing recipes using bitbake within layerindex-web
|
||||
#
|
||||
# Copyright (C) 2013 Intel Corporation
|
||||
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
||||
#
|
||||
# Licensed under the MIT license, see COPYING.MIT for details
|
||||
|
||||
import sys
|
||||
import os
|
||||
import os.path
|
||||
import utils
|
||||
import tempfile
|
||||
|
||||
class RecipeParseError(Exception):
|
||||
def __init__(self, msg):
|
||||
self.msg = msg
|
||||
|
||||
def __str__(self):
|
||||
return self.msg
|
||||
|
||||
def _setup_tinfoil(bitbakepath, enable_tracking):
|
||||
sys.path.insert(0, bitbakepath + '/lib')
|
||||
import bb.tinfoil
|
||||
import bb.cooker
|
||||
import bb.data
|
||||
tinfoil = bb.tinfoil.Tinfoil()
|
||||
if enable_tracking:
|
||||
tinfoil.cooker.enableDataTracking()
|
||||
tinfoil.prepare(config_only = True)
|
||||
|
||||
return tinfoil
|
||||
|
||||
def _parse_layer_conf(layerdir, data):
|
||||
data.setVar('LAYERDIR', str(layerdir))
|
||||
if hasattr(bb, "cookerdata"):
|
||||
# Newer BitBake
|
||||
data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data)
|
||||
else:
|
||||
# Older BitBake (1.18 and below)
|
||||
data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data)
|
||||
data.expandVarref('LAYERDIR')
|
||||
|
||||
|
||||
def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False):
|
||||
if not nocheckout:
|
||||
# Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc)
|
||||
out = utils.runcmd("git checkout origin/%s" % branch.bitbake_branch, bitbakepath)
|
||||
out = utils.runcmd("git clean -f -x", bitbakepath)
|
||||
|
||||
# Skip sanity checks
|
||||
os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS'
|
||||
os.environ['DISABLE_SANITY_CHECKS'] = '1'
|
||||
|
||||
fetchdir = settings.LAYER_FETCH_DIR
|
||||
|
||||
# Ensure we have OE-Core set up to get some base configuration
|
||||
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
|
||||
if not core_layer:
|
||||
raise RecipeParseError("Unable to find core layer %s in database; check CORE_LAYER_NAME setting" % settings.CORE_LAYER_NAME)
|
||||
core_layerbranch = core_layer.get_layerbranch(branch.name)
|
||||
core_branchname = branch.name
|
||||
if core_layerbranch:
|
||||
core_subdir = core_layerbranch.vcs_subdir
|
||||
if core_layerbranch.actual_branch:
|
||||
core_branchname = core_layerbranch.actual_branch
|
||||
else:
|
||||
core_subdir = 'meta'
|
||||
core_urldir = core_layer.get_fetch_dir()
|
||||
core_repodir = os.path.join(fetchdir, core_urldir)
|
||||
core_layerdir = os.path.join(core_repodir, core_subdir)
|
||||
if not nocheckout:
|
||||
out = utils.runcmd("git checkout origin/%s" % core_branchname, core_repodir)
|
||||
out = utils.runcmd("git clean -f -x", core_repodir)
|
||||
# The directory above where this script exists should contain our conf/layer.conf,
|
||||
# so add it to BBPATH along with the core layer directory
|
||||
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
||||
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir))
|
||||
|
||||
# Change into a temporary directory so we don't write the cache and other files to the current dir
|
||||
if not os.path.exists(settings.TEMP_BASE_DIR):
|
||||
os.makedirs(settings.TEMP_BASE_DIR)
|
||||
tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
|
||||
os.chdir(tempdir)
|
||||
|
||||
tinfoil = _setup_tinfoil(bitbakepath, enable_tracking)
|
||||
|
||||
# Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log)
|
||||
oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True)
|
||||
os.makedirs(oe_tmpdir)
|
||||
|
||||
return (tinfoil, tempdir)
|
||||
|
||||
def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch):
|
||||
# Parse layer.conf files for this layer and its dependencies
|
||||
# This is necessary not just because BBPATH needs to be set in order
|
||||
# for include/require/inherit to work outside of the current directory
|
||||
# or across layers, but also because custom variable values might be
|
||||
# set in layer.conf.
|
||||
config_data_copy = bb.data.createCopy(config_data)
|
||||
_parse_layer_conf(layerdir, config_data_copy)
|
||||
for dep in layerbranch.dependencies_set.all():
|
||||
depurldir = dep.dependency.get_fetch_dir()
|
||||
deprepodir = os.path.join(fetchdir, depurldir)
|
||||
deplayerbranch = dep.dependency.get_layerbranch(layerbranch.branch.name)
|
||||
if not deplayerbranch:
|
||||
raise RecipeParseError('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, layerbranch.branch.name))
|
||||
deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir)
|
||||
_parse_layer_conf(deplayerdir, config_data_copy)
|
||||
config_data_copy.delVar('LAYERDIR')
|
||||
return config_data_copy
|
||||
|
||||
|
|
@ -12,23 +12,16 @@ import sys
|
|||
import os.path
|
||||
import optparse
|
||||
import logging
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
import fnmatch
|
||||
import re
|
||||
import tempfile
|
||||
import shutil
|
||||
from distutils.version import LooseVersion
|
||||
import utils
|
||||
import recipeparse
|
||||
|
||||
def logger_create():
|
||||
logger = logging.getLogger("LayerIndexUpdate")
|
||||
loggerhandler = logging.StreamHandler()
|
||||
loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
||||
logger.addHandler(loggerhandler)
|
||||
logger.setLevel(logging.INFO)
|
||||
return logger
|
||||
|
||||
logger = logger_create()
|
||||
logger = utils.logger_create('LayerIndexUpdate')
|
||||
|
||||
# Ensure PythonGit is installed (buildhistory_analysis needs it)
|
||||
try:
|
||||
|
|
@ -38,27 +31,6 @@ except ImportError:
|
|||
sys.exit(1)
|
||||
|
||||
|
||||
def runcmd(cmd,destdir=None,printerr=True):
|
||||
"""
|
||||
execute command, raise CalledProcessError if fail
|
||||
return output if succeed
|
||||
"""
|
||||
logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
|
||||
out = os.tmpfile()
|
||||
try:
|
||||
subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
|
||||
except subprocess.CalledProcessError,e:
|
||||
out.seek(0)
|
||||
if printerr:
|
||||
logger.error("%s" % out.read())
|
||||
raise e
|
||||
|
||||
out.seek(0)
|
||||
output = out.read()
|
||||
logger.debug("output: %s" % output.rstrip() )
|
||||
return output
|
||||
|
||||
|
||||
machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
|
||||
bbclass_re = re.compile(r'classes/([^/.]*).bbclass$')
|
||||
def detect_file_type(path, subdir_start):
|
||||
|
|
@ -150,30 +122,6 @@ def update_machine_conf_file(path, machine):
|
|||
break
|
||||
machine.description = desc
|
||||
|
||||
def parse_layer_conf(layerdir, data):
|
||||
data.setVar('LAYERDIR', str(layerdir))
|
||||
if hasattr(bb, "cookerdata"):
|
||||
# Newer BitBake
|
||||
data = bb.cookerdata.parse_config_file(os.path.join(layerdir, "conf", "layer.conf"), data)
|
||||
else:
|
||||
# Older BitBake (1.18 and below)
|
||||
data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data)
|
||||
data.expandVarref('LAYERDIR')
|
||||
|
||||
def get_branch(branchname):
|
||||
from layerindex.models import Branch
|
||||
res = list(Branch.objects.filter(name=branchname)[:1])
|
||||
if res:
|
||||
return res[0]
|
||||
return None
|
||||
|
||||
def get_layer(layername):
|
||||
from layerindex.models import LayerItem
|
||||
res = list(LayerItem.objects.filter(name=layername)[:1])
|
||||
if res:
|
||||
return res[0]
|
||||
return None
|
||||
|
||||
def main():
|
||||
if LooseVersion(git.__version__) < '0.3.1':
|
||||
logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
|
||||
|
|
@ -216,22 +164,14 @@ def main():
|
|||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
# Get access to our Django model
|
||||
newpath = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])) + '/..')
|
||||
sys.path.append(newpath)
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
|
||||
|
||||
from django.core.management import setup_environ
|
||||
from django.conf import settings
|
||||
utils.setup_django()
|
||||
import settings
|
||||
from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
|
||||
from django.db import transaction
|
||||
import settings
|
||||
|
||||
setup_environ(settings)
|
||||
|
||||
logger.setLevel(options.loglevel)
|
||||
|
||||
branch = get_branch(options.branch)
|
||||
branch = utils.get_branch(options.branch)
|
||||
if not branch:
|
||||
logger.error("Specified branch %s is not valid" % options.branch)
|
||||
sys.exit(1)
|
||||
|
|
@ -270,9 +210,9 @@ def main():
|
|||
out = None
|
||||
try:
|
||||
if not os.path.exists(repodir):
|
||||
out = runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir)
|
||||
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir)
|
||||
else:
|
||||
out = runcmd("git fetch", repodir)
|
||||
out = utils.runcmd("git fetch", repodir)
|
||||
except Exception as e:
|
||||
logger.error("Fetch of layer %s failed: %s" % (layer.name, str(e)))
|
||||
failedrepos.append(layer.vcs_url)
|
||||
|
|
@ -285,54 +225,15 @@ def main():
|
|||
|
||||
logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL)
|
||||
if not os.path.exists(bitbakepath):
|
||||
out = runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir)
|
||||
out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir)
|
||||
else:
|
||||
out = runcmd("git fetch", bitbakepath)
|
||||
out = utils.runcmd("git fetch", bitbakepath)
|
||||
|
||||
if not options.nocheckout:
|
||||
# Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc)
|
||||
out = runcmd("git checkout origin/%s" % branch.bitbake_branch, bitbakepath)
|
||||
out = runcmd("git clean -f -x", bitbakepath)
|
||||
|
||||
# Skip sanity checks
|
||||
os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS'
|
||||
os.environ['DISABLE_SANITY_CHECKS'] = '1'
|
||||
|
||||
# Ensure we have OE-Core set up to get some base configuration
|
||||
core_layer = get_layer(settings.CORE_LAYER_NAME)
|
||||
if not core_layer:
|
||||
logger.error("Unable to find core layer %s in database; check CORE_LAYER_NAME setting" % settings.CORE_LAYER_NAME)
|
||||
try:
|
||||
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout)
|
||||
except recipeparse.RecipeParseError as e:
|
||||
logger.error(str(e))
|
||||
sys.exit(1)
|
||||
core_layerbranch = core_layer.get_layerbranch(options.branch)
|
||||
core_branchname = options.branch
|
||||
if core_layerbranch:
|
||||
core_subdir = core_layerbranch.vcs_subdir
|
||||
if core_layerbranch.actual_branch:
|
||||
core_branchname = core_layerbranch.actual_branch
|
||||
else:
|
||||
core_subdir = 'meta'
|
||||
core_urldir = core_layer.get_fetch_dir()
|
||||
core_repodir = os.path.join(fetchdir, core_urldir)
|
||||
core_layerdir = os.path.join(core_repodir, core_subdir)
|
||||
if not options.nocheckout:
|
||||
out = runcmd("git checkout origin/%s" % core_branchname, core_repodir)
|
||||
out = runcmd("git clean -f -x", core_repodir)
|
||||
# The directory above where this script exists should contain our conf/layer.conf,
|
||||
# so add it to BBPATH along with the core layer directory
|
||||
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
||||
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir))
|
||||
|
||||
# Change into a temporary directory so we don't write the cache and other files to the current dir
|
||||
if not os.path.exists(settings.TEMP_BASE_DIR):
|
||||
os.makedirs(settings.TEMP_BASE_DIR)
|
||||
tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
|
||||
os.chdir(tempdir)
|
||||
|
||||
sys.path.extend([bitbakepath + '/lib'])
|
||||
import bb.tinfoil
|
||||
import bb.cooker
|
||||
tinfoil = bb.tinfoil.Tinfoil()
|
||||
tinfoil.prepare(config_only = True)
|
||||
|
||||
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
|
||||
tinfoil.config_data.setVar('SUMMARY', '')
|
||||
|
|
@ -344,10 +245,6 @@ def main():
|
|||
# why won't they just fix that?!)
|
||||
tinfoil.config_data.setVar('LICENSE', '')
|
||||
|
||||
# Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log)
|
||||
oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True)
|
||||
os.makedirs(oe_tmpdir)
|
||||
|
||||
# Process and extract data from each layer
|
||||
for layer in layerquery:
|
||||
transaction.enter_transaction_management()
|
||||
|
|
@ -419,8 +316,8 @@ def main():
|
|||
if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
|
||||
# Check out appropriate branch
|
||||
if not options.nocheckout:
|
||||
out = runcmd("git checkout origin/%s" % branchname, repodir)
|
||||
out = runcmd("git clean -f -x", repodir)
|
||||
out = utils.runcmd("git checkout origin/%s" % branchname, repodir)
|
||||
out = utils.runcmd("git clean -f -x", repodir)
|
||||
|
||||
if not os.path.exists(layerdir):
|
||||
if options.branch == 'master':
|
||||
|
|
@ -439,25 +336,12 @@ def main():
|
|||
|
||||
logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
|
||||
|
||||
# Parse layer.conf files for this layer and its dependencies
|
||||
# This is necessary not just because BBPATH needs to be set in order
|
||||
# for include/require/inherit to work outside of the current directory
|
||||
# or across layers, but also because custom variable values might be
|
||||
# set in layer.conf.
|
||||
|
||||
config_data_copy = bb.data.createCopy(tinfoil.config_data)
|
||||
parse_layer_conf(layerdir, config_data_copy)
|
||||
for dep in layerbranch.dependencies_set.all():
|
||||
depurldir = dep.dependency.get_fetch_dir()
|
||||
deprepodir = os.path.join(fetchdir, depurldir)
|
||||
deplayerbranch = dep.dependency.get_layerbranch(options.branch)
|
||||
if not deplayerbranch:
|
||||
logger.error('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, options.branch))
|
||||
try:
|
||||
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
|
||||
except recipeparse.RecipeParseError as e:
|
||||
logger.error(str(e))
|
||||
transaction.rollback()
|
||||
continue
|
||||
deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir)
|
||||
parse_layer_conf(deplayerdir, config_data_copy)
|
||||
config_data_copy.delVar('LAYERDIR')
|
||||
|
||||
if layerbranch.vcs_last_rev and not options.reload:
|
||||
try:
|
||||
|
|
|
|||
65
layerindex/utils.py
Normal file
65
layerindex/utils.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
# Utilities for layerindex-web
|
||||
#
|
||||
# Copyright (C) 2013 Intel Corporation
|
||||
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
||||
#
|
||||
# Licensed under the MIT license, see COPYING.MIT for details
|
||||
|
||||
import sys
|
||||
import os.path
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
def get_branch(branchname):
|
||||
from layerindex.models import Branch
|
||||
res = list(Branch.objects.filter(name=branchname)[:1])
|
||||
if res:
|
||||
return res[0]
|
||||
return None
|
||||
|
||||
def get_layer(layername):
|
||||
from layerindex.models import LayerItem
|
||||
res = list(LayerItem.objects.filter(name=layername)[:1])
|
||||
if res:
|
||||
return res[0]
|
||||
return None
|
||||
|
||||
def runcmd(cmd, destdir=None, printerr=True):
|
||||
"""
|
||||
execute command, raise CalledProcessError if fail
|
||||
return output if succeed
|
||||
"""
|
||||
#logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
|
||||
out = os.tmpfile()
|
||||
try:
|
||||
subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
|
||||
except subprocess.CalledProcessError,e:
|
||||
out.seek(0)
|
||||
if printerr:
|
||||
logger.error("%s" % out.read())
|
||||
raise e
|
||||
|
||||
out.seek(0)
|
||||
output = out.read()
|
||||
#logger.debug("output: %s" % output.rstrip() )
|
||||
return output
|
||||
|
||||
def setup_django():
|
||||
# Get access to our Django model
|
||||
newpath = os.path.abspath(os.path.dirname(__file__) + '/..')
|
||||
sys.path.append(newpath)
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
|
||||
|
||||
from django.core.management import setup_environ
|
||||
from django.conf import settings
|
||||
import settings
|
||||
|
||||
setup_environ(settings)
|
||||
|
||||
def logger_create(name):
|
||||
logger = logging.getLogger(name)
|
||||
loggerhandler = logging.StreamHandler()
|
||||
loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
||||
logger.addHandler(loggerhandler)
|
||||
logger.setLevel(logging.INFO)
|
||||
return logger
|
||||
Loading…
Reference in New Issue
Block a user