mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 12:49:01 +02:00
3cc0b78935
If we're creating a LayerBranch record, it's helpful to copy the maintainer and dependency records in from master as they're usually going to be the same; if they're not the user can always change them afterwards. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
522 lines
22 KiB
Python
Executable File
522 lines
22 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Fetch layer repositories and update layer index database
|
|
#
|
|
# Copyright (C) 2013 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
|
|
import sys
|
|
import os.path
|
|
import optparse
|
|
import logging
|
|
import subprocess
|
|
from datetime import datetime
|
|
import fnmatch
|
|
import re
|
|
import tempfile
|
|
import shutil
|
|
from distutils.version import LooseVersion
|
|
|
|
def logger_create():
|
|
logger = logging.getLogger("LayerIndexUpdate")
|
|
loggerhandler = logging.StreamHandler()
|
|
loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
|
logger.addHandler(loggerhandler)
|
|
logger.setLevel(logging.INFO)
|
|
return logger
|
|
|
|
logger = logger_create()
|
|
|
|
# Ensure PythonGit is installed (buildhistory_analysis needs it)
|
|
try:
|
|
import git
|
|
except ImportError:
|
|
logger.error("Please install PythonGit 0.3.1 or later in order to use this script")
|
|
sys.exit(1)
|
|
|
|
|
|
def runcmd(cmd,destdir=None,printerr=True):
|
|
"""
|
|
execute command, raise CalledProcessError if fail
|
|
return output if succeed
|
|
"""
|
|
logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
|
|
out = os.tmpfile()
|
|
try:
|
|
subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
|
|
except subprocess.CalledProcessError,e:
|
|
out.seek(0)
|
|
if printerr:
|
|
logger.error("%s" % out.read())
|
|
raise e
|
|
|
|
out.seek(0)
|
|
output = out.read()
|
|
logger.debug("output: %s" % output.rstrip() )
|
|
return output
|
|
|
|
def sanitise_path(inpath):
|
|
outpath = ""
|
|
for c in inpath:
|
|
if c in '/ .=+?:':
|
|
outpath += "_"
|
|
else:
|
|
outpath += c
|
|
return outpath
|
|
|
|
|
|
def split_bb_file_path(recipe_path, subdir_start):
|
|
if fnmatch.fnmatch(recipe_path, "*.bb"):
|
|
if subdir_start:
|
|
filepath = os.path.relpath(os.path.dirname(recipe_path), subdir_start)
|
|
else:
|
|
filepath = os.path.dirname(recipe_path)
|
|
return (filepath, os.path.basename(recipe_path))
|
|
return (None, None)
|
|
|
|
conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
|
|
def check_machine_conf(path):
|
|
res = conf_re.search(path)
|
|
if res:
|
|
return res.group(1)
|
|
return None
|
|
|
|
def update_recipe_file(data, path, recipe, layerdir_start, repodir):
|
|
fn = str(os.path.join(path, recipe.filename))
|
|
try:
|
|
logger.debug('Updating recipe %s' % fn)
|
|
envdata = bb.cache.Cache.loadDataFull(fn, [], data)
|
|
envdata.setVar('SRCPV', 'X')
|
|
recipe.pn = envdata.getVar("PN", True)
|
|
recipe.pv = envdata.getVar("PV", True)
|
|
recipe.summary = envdata.getVar("SUMMARY", True)
|
|
recipe.description = envdata.getVar("DESCRIPTION", True)
|
|
recipe.section = envdata.getVar("SECTION", True)
|
|
recipe.license = envdata.getVar("LICENSE", True)
|
|
recipe.homepage = envdata.getVar("HOMEPAGE", True)
|
|
recipe.save()
|
|
|
|
# Get file dependencies within this layer
|
|
deps = envdata.getVar('__depends', True)
|
|
filedeps = []
|
|
for depstr, date in deps:
|
|
found = False
|
|
if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
|
|
filedeps.append(os.path.relpath(depstr, repodir))
|
|
from layerindex.models import RecipeFileDependency
|
|
RecipeFileDependency.objects.filter(recipe=recipe).delete()
|
|
for filedep in filedeps:
|
|
recipedep = RecipeFileDependency()
|
|
recipedep.layerbranch = recipe.layerbranch
|
|
recipedep.recipe = recipe
|
|
recipedep.path = filedep
|
|
recipedep.save()
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except BaseException as e:
|
|
logger.info("Unable to read %s: %s", fn, str(e))
|
|
|
|
def update_machine_conf_file(path, machine):
|
|
logger.debug('Updating machine %s' % path)
|
|
with open(path, 'r') as f:
|
|
for line in f:
|
|
if line.startswith('#@DESCRIPTION:'):
|
|
desc = line[14:].strip()
|
|
desc = re.sub(r'Machine configuration for (the )*', '', desc)
|
|
machine.description = desc
|
|
break
|
|
|
|
def parse_layer_conf(layerdir, data):
|
|
data.setVar('LAYERDIR', str(layerdir))
|
|
data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data)
|
|
data.expandVarref('LAYERDIR')
|
|
|
|
def get_branch(branchname):
|
|
from layerindex.models import Branch
|
|
res = list(Branch.objects.filter(name=branchname)[:1])
|
|
if res:
|
|
return res[0]
|
|
return None
|
|
|
|
def get_layer(layername):
|
|
from layerindex.models import LayerItem
|
|
res = list(LayerItem.objects.filter(name=layername)[:1])
|
|
if res:
|
|
return res[0]
|
|
return None
|
|
|
|
def main():
|
|
if LooseVersion(git.__version__) < '0.3.1':
|
|
logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
|
|
sys.exit(1)
|
|
|
|
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options]""")
|
|
|
|
parser.add_option("-b", "--branch",
|
|
help = "Specify branch to update",
|
|
action="store", dest="branch", default='master')
|
|
parser.add_option("-l", "--layer",
|
|
help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
|
|
action="store", dest="layers")
|
|
parser.add_option("-r", "--reload",
|
|
help = "Discard existing recipe data and fetch it from scratch",
|
|
action="store_true", dest="reload")
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("-x", "--nofetch",
|
|
help = "Don't fetch repositories",
|
|
action="store_true", dest="nofetch")
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
# Get access to our Django model
|
|
newpath = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0])) + '/..')
|
|
sys.path.append(newpath)
|
|
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
|
|
|
|
from django.core.management import setup_environ
|
|
from django.conf import settings
|
|
from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine
|
|
from django.db import transaction
|
|
import settings
|
|
|
|
setup_environ(settings)
|
|
|
|
branch = get_branch(options.branch)
|
|
if not branch:
|
|
logger.error("Specified branch %s is not valid" % options.branch)
|
|
sys.exit(1)
|
|
|
|
fetchdir = settings.LAYER_FETCH_DIR
|
|
if not fetchdir:
|
|
logger.error("Please set LAYER_FETCH_DIR in settings.py")
|
|
sys.exit(1)
|
|
|
|
if options.layers:
|
|
layerquery = LayerItem.objects.filter(name__in=options.layers.split(','))
|
|
if layerquery.count() == 0:
|
|
logger.error('No layers matching specified query "%s"' % options.layers)
|
|
sys.exit(1)
|
|
else:
|
|
layerquery = LayerItem.objects.filter(status='P')
|
|
if layerquery.count() == 0:
|
|
logger.info("No published layers to update")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(fetchdir):
|
|
os.makedirs(fetchdir)
|
|
fetchedrepos = []
|
|
failedrepos = []
|
|
|
|
bitbakepath = os.path.join(fetchdir, 'bitbake')
|
|
|
|
if not options.nofetch:
|
|
# Fetch latest metadata from repositories
|
|
for layer in layerquery:
|
|
# Handle multiple layers in a single repo
|
|
urldir = sanitise_path(layer.vcs_url)
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
if not layer.vcs_url in fetchedrepos:
|
|
logger.info("Fetching remote repository %s" % layer.vcs_url)
|
|
out = None
|
|
try:
|
|
if not os.path.exists(repodir):
|
|
out = runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir)
|
|
else:
|
|
out = runcmd("git fetch", repodir)
|
|
except Exception as e:
|
|
logger.error("fetch failed: %s" % str(e))
|
|
failedrepos.append(layer.vcs_url)
|
|
continue
|
|
fetchedrepos.append(layer.vcs_url)
|
|
|
|
logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL)
|
|
if not os.path.exists(bitbakepath):
|
|
out = runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir)
|
|
else:
|
|
out = runcmd("git fetch", bitbakepath)
|
|
|
|
# Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc)
|
|
out = runcmd("git checkout origin/%s" % branch.bitbake_branch, bitbakepath)
|
|
out = runcmd("git clean -f -x", bitbakepath)
|
|
|
|
# Skip sanity checks
|
|
os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS'
|
|
os.environ['DISABLE_SANITY_CHECKS'] = '1'
|
|
|
|
# Ensure we have OE-Core set up to get some base configuration
|
|
core_layer = get_layer(settings.CORE_LAYER_NAME)
|
|
if not core_layer:
|
|
logger.error("Unable to find core layer %s in database; check CORE_LAYER_NAME setting" % settings.CORE_LAYER_NAME)
|
|
sys.exit(1)
|
|
core_layerbranch = core_layer.get_layerbranch(options.branch)
|
|
if core_layerbranch:
|
|
core_subdir = core_layerbranch.vcs_subdir
|
|
else:
|
|
core_subdir = 'meta'
|
|
core_urldir = sanitise_path(core_layer.vcs_url)
|
|
core_repodir = os.path.join(fetchdir, core_urldir)
|
|
core_layerdir = os.path.join(core_repodir, core_subdir)
|
|
out = runcmd("git checkout origin/%s" % options.branch, core_repodir)
|
|
out = runcmd("git clean -f -x", core_repodir)
|
|
# The directory above where this script exists should contain our conf/layer.conf,
|
|
# so add it to BBPATH along with the core layer directory
|
|
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
|
|
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir))
|
|
|
|
# Change into a temporary directory so we don't write the cache and other files to the current dir
|
|
if not os.path.exists(settings.TEMP_BASE_DIR):
|
|
os.makedirs(settings.TEMP_BASE_DIR)
|
|
tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
|
|
os.chdir(tempdir)
|
|
|
|
sys.path.extend([bitbakepath + '/lib'])
|
|
import bb.tinfoil
|
|
import bb.cooker
|
|
tinfoil = bb.tinfoil.Tinfoil()
|
|
tinfoil.prepare(config_only = True)
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
|
|
tinfoil.config_data.setVar('SUMMARY', '')
|
|
# Clear the default value of DESCRIPTION so that we can see where it's not set
|
|
tinfoil.config_data.setVar('DESCRIPTION', '')
|
|
# Clear the default value of HOMEPAGE ('unknown')
|
|
tinfoil.config_data.setVar('HOMEPAGE', '')
|
|
# Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
|
|
# why won't they just fix that?!)
|
|
tinfoil.config_data.setVar('LICENSE', '')
|
|
|
|
# Process and extract data from each layer
|
|
for layer in layerquery:
|
|
transaction.enter_transaction_management()
|
|
transaction.managed(True)
|
|
try:
|
|
urldir = sanitise_path(layer.vcs_url)
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
if layer.vcs_url in failedrepos:
|
|
logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
|
|
transaction.rollback()
|
|
continue
|
|
|
|
# Collect repo info
|
|
repo = git.Repo(repodir)
|
|
assert repo.bare == False
|
|
try:
|
|
topcommit = repo.commit('origin/%s' % options.branch)
|
|
except:
|
|
logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, options.branch))
|
|
transaction.rollback()
|
|
continue
|
|
|
|
layerbranch = layer.get_layerbranch(options.branch)
|
|
if not layerbranch:
|
|
# LayerBranch doesn't exist for this branch, create it
|
|
layerbranch = LayerBranch()
|
|
layerbranch.layer = layer
|
|
layerbranch.branch = branch
|
|
layerbranch_master = layer.get_layerbranch('master')
|
|
if layerbranch_master:
|
|
layerbranch.vcs_subdir = layerbranch_master.vcs_subdir
|
|
layerbranch.save()
|
|
if layerbranch_master:
|
|
for maintainer in layerbranch_master.layermaintainer_set.all():
|
|
maintainer.pk = None
|
|
maintainer.id = None
|
|
maintainer.layerbranch = layerbranch
|
|
maintainer.save()
|
|
for dep in layerbranch_master.dependencies_set.all():
|
|
dep.pk = None
|
|
dep.id = None
|
|
dep.layerbranch = layerbranch
|
|
dep.save()
|
|
|
|
if layerbranch.vcs_subdir:
|
|
# Find latest commit in subdirectory
|
|
# A bit odd to do it this way but apparently there's no other way in the GitPython API
|
|
for commit in repo.iter_commits('origin/%s' % options.branch, paths=layerbranch.vcs_subdir):
|
|
topcommit = commit
|
|
break
|
|
|
|
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
|
layerdir_start = os.path.normpath(layerdir) + os.sep
|
|
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
|
|
layermachines = Machine.objects.filter(layerbranch=layerbranch)
|
|
if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
|
|
# Check out appropriate branch
|
|
out = runcmd("git checkout origin/%s" % options.branch, repodir)
|
|
|
|
if not os.path.exists(layerdir):
|
|
if options.branch == 'master':
|
|
logger.error("Subdirectory for layer %s does not exist on master branch!" % layer.name)
|
|
transaction.rollback()
|
|
continue
|
|
else:
|
|
logger.info("Skipping update of layer %s for branch %s - subdirectory does not exist on this branch" % (layer.name, options.branch))
|
|
transaction.rollback()
|
|
continue
|
|
|
|
logger.info("Collecting data for layer %s on branch %s" % (layer.name, options.branch))
|
|
|
|
# Parse layer.conf files for this layer and its dependencies
|
|
# This is necessary not just because BBPATH needs to be set in order
|
|
# for include/require/inherit to work outside of the current directory
|
|
# or across layers, but also because custom variable values might be
|
|
# set in layer.conf.
|
|
|
|
config_data_copy = bb.data.createCopy(tinfoil.config_data)
|
|
parse_layer_conf(layerdir, config_data_copy)
|
|
for dep in layerbranch.dependencies_set.all():
|
|
depurldir = sanitise_path(dep.dependency.vcs_url)
|
|
deprepodir = os.path.join(fetchdir, depurldir)
|
|
deplayerbranch = dep.dependency.get_layerbranch(options.branch)
|
|
if not deplayerbranch:
|
|
logger.error('Dependency %s of layer %s does not have branch record for branch %s' % (dep.dependency.name, layer.name, options.branch))
|
|
transaction.rollback()
|
|
continue
|
|
deplayerdir = os.path.join(deprepodir, deplayerbranch.vcs_subdir)
|
|
parse_layer_conf(deplayerdir, config_data_copy)
|
|
config_data_copy.delVar('LAYERDIR')
|
|
|
|
if layerbranch.vcs_last_rev and not options.reload:
|
|
try:
|
|
diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
|
|
except Exception as e:
|
|
logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
|
|
diff = None
|
|
else:
|
|
diff = None
|
|
|
|
if diff:
|
|
# Apply git changes to existing recipe list
|
|
|
|
if layerbranch.vcs_subdir:
|
|
subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
|
|
else:
|
|
subdir_start = ""
|
|
|
|
updatedrecipes = set()
|
|
for d in diff.iter_change_type('D'):
|
|
path = d.a_blob.path
|
|
if path.startswith(subdir_start):
|
|
(filepath, filename) = split_bb_file_path(path, subdir_start)
|
|
if filename:
|
|
layerrecipes.filter(filepath=filepath).filter(filename=filename).delete()
|
|
else:
|
|
machinename = check_machine_conf(path)
|
|
if machinename:
|
|
layermachines.filter(name=machinename).delete()
|
|
|
|
for d in diff.iter_change_type('A'):
|
|
path = d.b_blob.path
|
|
if path.startswith(subdir_start):
|
|
(filepath, filename) = split_bb_file_path(path, subdir_start)
|
|
if filename:
|
|
recipe = Recipe()
|
|
recipe.layerbranch = layerbranch
|
|
recipe.filename = filename
|
|
recipe.filepath = filepath
|
|
update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
|
|
recipe.save()
|
|
updatedrecipes.add(recipe)
|
|
else:
|
|
machinename = check_machine_conf(path)
|
|
if machinename:
|
|
machine = Machine()
|
|
machine.layerbranch = layerbranch
|
|
machine.name = machinename
|
|
update_machine_conf_file(os.path.join(repodir, path), machine)
|
|
machine.save()
|
|
|
|
dirtyrecipes = set()
|
|
for d in diff.iter_change_type('M'):
|
|
path = d.a_blob.path
|
|
if path.startswith(subdir_start):
|
|
(filepath, filename) = split_bb_file_path(path, subdir_start)
|
|
if filename:
|
|
results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
|
|
if results:
|
|
recipe = results[0]
|
|
update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
|
|
recipe.save()
|
|
updatedrecipes.add(recipe)
|
|
else:
|
|
machinename = check_machine_conf(path)
|
|
if machinename:
|
|
results = layermachines.filter(name=machinename)
|
|
if results:
|
|
machine = results[0]
|
|
update_machine_conf_file(os.path.join(repodir, path), machine)
|
|
machine.save()
|
|
deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
|
|
for dep in deps:
|
|
dirtyrecipes.add(dep.recipe)
|
|
|
|
dirtyrecipes -= updatedrecipes
|
|
for recipe in dirtyrecipes:
|
|
update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
|
|
else:
|
|
# Collect recipe data from scratch
|
|
layerrecipes.delete()
|
|
layermachines.delete()
|
|
for root, dirs, files in os.walk(layerdir):
|
|
for f in files:
|
|
if fnmatch.fnmatch(f, "*.bb"):
|
|
recipe = Recipe()
|
|
recipe.layerbranch = layerbranch
|
|
recipe.filename = f
|
|
recipe.filepath = os.path.relpath(root, layerdir)
|
|
update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
|
|
recipe.save()
|
|
else:
|
|
fullpath = os.path.join(root, f)
|
|
machinename = check_machine_conf(fullpath)
|
|
if machinename:
|
|
machine = Machine()
|
|
machine.layerbranch = layerbranch
|
|
machine.name = machinename
|
|
update_machine_conf_file(fullpath, machine)
|
|
machine.save()
|
|
|
|
# Save repo info
|
|
layerbranch.vcs_last_rev = topcommit.hexsha
|
|
layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
|
|
else:
|
|
logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, options.branch))
|
|
|
|
layerbranch.vcs_last_fetch = datetime.now()
|
|
layerbranch.save()
|
|
|
|
if options.dryrun:
|
|
transaction.rollback()
|
|
else:
|
|
transaction.commit()
|
|
except:
|
|
import traceback
|
|
traceback.print_exc()
|
|
transaction.rollback()
|
|
finally:
|
|
transaction.leave_transaction_management()
|
|
|
|
shutil.rmtree(tempdir)
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|