mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 03:49:10 +02:00
15748efcc1
Fix a couple of problems with shutting down tinfoil: 1) tinfoil.shutdown() wasn't being called on error because it wasn't inside a finally: section, thus on error the script hung with bitbake master (since the tinfoil2 changes) - this is almost certainly a bug in bitbake but let's handle it here anyway. 2) We need to check whether the tinfoil object actually has a shutdown attribute since we still want to support updating for older branches where tinfoil didn't have a shutdown() method. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
691 lines
34 KiB
Python
691 lines
34 KiB
Python
#!/usr/bin/env python
|
|
|
|
# Update layer index database for a single layer
|
|
#
|
|
# Copyright (C) 2013-2016 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
|
|
import sys
|
|
import os
|
|
import optparse
|
|
import logging
|
|
from datetime import datetime
|
|
import re
|
|
import tempfile
|
|
import shutil
|
|
from distutils.version import LooseVersion
|
|
import itertools
|
|
import utils
|
|
import recipeparse
|
|
|
|
import warnings
|
|
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
|
|
logger = utils.logger_create('LayerIndexUpdate')
|
|
|
|
# Ensure PythonGit is installed (buildhistory_analysis needs it)
|
|
try:
|
|
import git
|
|
except ImportError:
|
|
logger.error("Please install PythonGit 0.3.1 or later in order to use this script")
|
|
sys.exit(1)
|
|
|
|
|
|
class DryRunRollbackException(Exception):
|
|
pass
|
|
|
|
|
|
def check_machine_conf(path, subdir_start):
|
|
subpath = path[len(subdir_start):]
|
|
res = conf_re.match(subpath)
|
|
if res:
|
|
return res.group(1)
|
|
return None
|
|
|
|
def split_recipe_fn(path):
|
|
splitfn = os.path.basename(path).split('.bb')[0].split('_', 2)
|
|
pn = splitfn[0]
|
|
if len(splitfn) > 1:
|
|
pv = splitfn[1]
|
|
else:
|
|
pv = "1.0"
|
|
return (pn, pv)
|
|
|
|
def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir):
|
|
fn = str(os.path.join(path, recipe.filename))
|
|
try:
|
|
logger.debug('Updating recipe %s' % fn)
|
|
if hasattr(tinfoil, 'parse_recipe_file'):
|
|
envdata = tinfoil.parse_recipe_file(fn, appends=False, config_data=data)
|
|
else:
|
|
envdata = bb.cache.Cache.loadDataFull(fn, [], data)
|
|
envdata.setVar('SRCPV', 'X')
|
|
recipe.pn = envdata.getVar("PN", True)
|
|
recipe.pv = envdata.getVar("PV", True)
|
|
recipe.summary = envdata.getVar("SUMMARY", True)
|
|
recipe.description = envdata.getVar("DESCRIPTION", True)
|
|
recipe.section = envdata.getVar("SECTION", True)
|
|
recipe.license = envdata.getVar("LICENSE", True)
|
|
recipe.homepage = envdata.getVar("HOMEPAGE", True)
|
|
recipe.bugtracker = envdata.getVar("BUGTRACKER", True) or ""
|
|
recipe.provides = envdata.getVar("PROVIDES", True) or ""
|
|
recipe.bbclassextend = envdata.getVar("BBCLASSEXTEND", True) or ""
|
|
# Handle recipe inherits for this recipe
|
|
gr = set(data.getVar("__inherit_cache", True) or [])
|
|
lr = set(envdata.getVar("__inherit_cache", True) or [])
|
|
recipe.inherits = ' '.join(sorted({os.path.splitext(os.path.basename(r))[0] for r in lr if r not in gr}))
|
|
recipe.blacklisted = envdata.getVarFlag('PNBLACKLIST', recipe.pn, True) or ""
|
|
recipe.save()
|
|
|
|
# Get file dependencies within this layer
|
|
deps = envdata.getVar('__depends', True)
|
|
filedeps = []
|
|
for depstr, date in deps:
|
|
found = False
|
|
if depstr.startswith(layerdir_start) and not depstr.endswith('/conf/layer.conf'):
|
|
filedeps.append(os.path.relpath(depstr, repodir))
|
|
from layerindex.models import RecipeFileDependency
|
|
|
|
recipedeps_delete = []
|
|
|
|
recipedeps = RecipeFileDependency.objects.filter(recipe=recipe)
|
|
|
|
for values in recipedeps.values('path'):
|
|
if 'path' in values:
|
|
recipedeps_delete.append(values['path'])
|
|
|
|
for filedep in filedeps:
|
|
if filedep in recipedeps_delete:
|
|
recipedeps_delete.remove(filedep)
|
|
continue
|
|
# New item, add it...
|
|
recipedep = RecipeFileDependency()
|
|
recipedep.layerbranch = recipe.layerbranch
|
|
recipedep.recipe = recipe
|
|
recipedep.path = filedep
|
|
recipedep.save()
|
|
|
|
for filedep in recipedeps_delete:
|
|
logger.debug('%s: removing %s' % (recipe.layerbranch, filedep))
|
|
recipedeps.filter(path=filedep).delete()
|
|
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except BaseException as e:
|
|
if not recipe.pn:
|
|
recipe.pn = recipe.filename[:-3].split('_')[0]
|
|
logger.error("Unable to read %s: %s", fn, str(e))
|
|
|
|
def update_machine_conf_file(path, machine):
|
|
logger.debug('Updating machine %s' % path)
|
|
desc = ""
|
|
with open(path, 'r') as f:
|
|
for line in f:
|
|
if line.startswith('#@NAME:'):
|
|
desc = line[7:].strip()
|
|
if line.startswith('#@DESCRIPTION:'):
|
|
desc = line[14:].strip()
|
|
desc = re.sub(r'Machine configuration for( running)*( an)*( the)*', '', desc)
|
|
break
|
|
machine.description = desc
|
|
|
|
def update_distro_conf_file(path, distro):
|
|
logger.debug('Updating distro %s' % path)
|
|
desc = ""
|
|
with open(path, 'r') as f:
|
|
for line in f:
|
|
if line.startswith('#@NAME:'):
|
|
desc = line[7:].strip()
|
|
if line.startswith('#@DESCRIPTION:'):
|
|
desc = line[14:].strip()
|
|
desc = re.sub(r'Distribution configuration for( running)*( an)*( the)*', '', desc)
|
|
break
|
|
distro.description = desc
|
|
|
|
def main():
|
|
if LooseVersion(git.__version__) < '0.3.1':
|
|
logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
|
|
sys.exit(1)
|
|
|
|
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options]""")
|
|
|
|
parser.add_option("-b", "--branch",
|
|
help = "Specify branch to update",
|
|
action="store", dest="branch", default='master')
|
|
parser.add_option("-l", "--layer",
|
|
help = "Layer to update",
|
|
action="store", dest="layer")
|
|
parser.add_option("-r", "--reload",
|
|
help = "Reload recipe data instead of updating since last update",
|
|
action="store_true", dest="reload")
|
|
parser.add_option("", "--fullreload",
|
|
help = "Discard existing recipe data and fetch it from scratch",
|
|
action="store_true", dest="fullreload")
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("", "--nocheckout",
|
|
help = "Don't check out branches",
|
|
action="store_true", dest="nocheckout")
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
if len(args) > 1:
|
|
logger.error('unexpected argument "%s"' % args[1])
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
if options.fullreload:
|
|
options.reload = True
|
|
|
|
utils.setup_django()
|
|
import settings
|
|
from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass
|
|
from django.db import transaction
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
branch = utils.get_branch(options.branch)
|
|
if not branch:
|
|
logger.error("Specified branch %s is not valid" % options.branch)
|
|
sys.exit(1)
|
|
|
|
fetchdir = settings.LAYER_FETCH_DIR
|
|
if not fetchdir:
|
|
logger.error("Please set LAYER_FETCH_DIR in settings.py")
|
|
sys.exit(1)
|
|
|
|
bitbakepath = os.path.join(fetchdir, 'bitbake')
|
|
|
|
try:
|
|
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
|
|
except recipeparse.RecipeParseError as e:
|
|
logger.error(str(e))
|
|
sys.exit(1)
|
|
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
|
|
tinfoil.config_data.setVar('SUMMARY', '')
|
|
# Clear the default value of DESCRIPTION so that we can see where it's not set
|
|
tinfoil.config_data.setVar('DESCRIPTION', '')
|
|
# Clear the default value of HOMEPAGE ('unknown')
|
|
tinfoil.config_data.setVar('HOMEPAGE', '')
|
|
# Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
|
|
# why won't they just fix that?!)
|
|
tinfoil.config_data.setVar('LICENSE', '')
|
|
|
|
try:
|
|
with transaction.atomic():
|
|
layer = utils.get_layer(options.layer)
|
|
urldir = layer.get_fetch_dir()
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
|
|
layerbranch = layer.get_layerbranch(options.branch)
|
|
|
|
branchname = options.branch
|
|
branchdesc = options.branch
|
|
if layerbranch:
|
|
if layerbranch.actual_branch:
|
|
branchname = layerbranch.actual_branch
|
|
branchdesc = "%s (%s)" % (options.branch, branchname)
|
|
|
|
# Collect repo info
|
|
repo = git.Repo(repodir)
|
|
assert repo.bare == False
|
|
try:
|
|
if options.nocheckout:
|
|
topcommit = repo.commit('HEAD')
|
|
else:
|
|
topcommit = repo.commit('origin/%s' % branchname)
|
|
except:
|
|
if layerbranch:
|
|
logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
|
|
else:
|
|
logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
|
|
sys.exit(1)
|
|
|
|
newbranch = False
|
|
if not layerbranch:
|
|
# LayerBranch doesn't exist for this branch, create it
|
|
newbranch = True
|
|
layerbranch = LayerBranch()
|
|
layerbranch.layer = layer
|
|
layerbranch.branch = branch
|
|
layerbranch_source = layer.get_layerbranch(branch)
|
|
if not layerbranch_source:
|
|
layerbranch_source = layer.get_layerbranch(None)
|
|
if layerbranch_source:
|
|
layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
|
|
layerbranch.save()
|
|
if layerbranch_source:
|
|
for maintainer in layerbranch_source.layermaintainer_set.all():
|
|
maintainer.pk = None
|
|
maintainer.id = None
|
|
maintainer.layerbranch = layerbranch
|
|
maintainer.save()
|
|
|
|
if layerbranch.vcs_subdir and not options.nocheckout:
|
|
# Find latest commit in subdirectory
|
|
# A bit odd to do it this way but apparently there's no other way in the GitPython API
|
|
topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
|
|
if not topcommit:
|
|
# This will error out if the directory is completely invalid or had never existed at this point
|
|
# If it previously existed but has since been deleted, you will get the revision where it was
|
|
# deleted - so we need to handle that case separately later
|
|
if newbranch:
|
|
logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
|
|
elif layerbranch.vcs_subdir:
|
|
logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
|
|
else:
|
|
logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
|
|
sys.exit(1)
|
|
|
|
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
|
layerdir_start = os.path.normpath(layerdir) + os.sep
|
|
|
|
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
|
|
layermachines = Machine.objects.filter(layerbranch=layerbranch)
|
|
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
|
|
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
|
|
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
|
|
if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
|
|
# Check out appropriate branch
|
|
if not options.nocheckout:
|
|
out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
|
|
out = utils.runcmd("git clean -f -x", repodir, logger=logger)
|
|
|
|
if layerbranch.vcs_subdir and not os.path.exists(layerdir):
|
|
if newbranch:
|
|
logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
|
|
else:
|
|
logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
|
|
logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
|
|
sys.exit(1)
|
|
|
|
logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
|
|
|
|
from layerconfparse import LayerConfParse
|
|
layerconfparser = LayerConfParse(logger=logger, tinfoil=tinfoil)
|
|
layer_config_data = layerconfparser.parse_layer(layerbranch, layerdir)
|
|
if not layer_config_data:
|
|
logger.info("Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc))
|
|
layerconfparser.shutdown()
|
|
sys.exit(1)
|
|
utils.add_dependencies(layerbranch, layer_config_data, logger=logger)
|
|
utils.add_recommends(layerbranch, layer_config_data, logger=logger)
|
|
utils.set_layerbranch_collection_version(layerbranch, layer_config_data, logger=logger)
|
|
layerbranch.save()
|
|
|
|
try:
|
|
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
|
|
except recipeparse.RecipeParseError as e:
|
|
logger.error(str(e))
|
|
sys.exit(1)
|
|
|
|
if layerbranch.vcs_last_rev and not options.reload:
|
|
try:
|
|
diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
|
|
except Exception as e:
|
|
logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
|
|
diff = None
|
|
else:
|
|
diff = None
|
|
|
|
# We handle recipes specially to try to preserve the same id
|
|
# when recipe upgrades happen (so that if a user bookmarks a
|
|
# recipe page it remains valid)
|
|
layerrecipes_delete = []
|
|
layerrecipes_add = []
|
|
|
|
# Check if any paths should be ignored because there are layers within this layer
|
|
removedirs = []
|
|
for root, dirs, files in os.walk(layerdir):
|
|
for diritem in dirs:
|
|
if os.path.exists(os.path.join(root, diritem, 'conf', 'layer.conf')):
|
|
removedirs.append(os.path.join(root, diritem) + os.sep)
|
|
|
|
if diff:
|
|
# Apply git changes to existing recipe list
|
|
|
|
if layerbranch.vcs_subdir:
|
|
subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
|
|
else:
|
|
subdir_start = ""
|
|
|
|
updatedrecipes = set()
|
|
dirtyrecipes = set()
|
|
other_deletes = []
|
|
other_adds = []
|
|
for diffitem in diff.iter_change_type('R'):
|
|
oldpath = diffitem.a_blob.path
|
|
newpath = diffitem.b_blob.path
|
|
skip = False
|
|
for removedir in removedirs:
|
|
# FIXME what about files moved into removedirs?
|
|
if oldpath.startswith(removedir):
|
|
skip = True
|
|
break
|
|
if skip:
|
|
continue
|
|
if oldpath.startswith(subdir_start):
|
|
(oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type(oldpath, subdir_start)
|
|
(newtypename, newfilepath, newfilename) = recipeparse.detect_file_type(newpath, subdir_start)
|
|
if oldtypename != newtypename:
|
|
# This is most likely to be a .inc file renamed to a .bb - and since
|
|
# there may be another recipe deleted at the same time we probably want
|
|
# to consider that, so just treat it as a delete and an add
|
|
logger.debug("Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath))
|
|
other_deletes.append(diffitem)
|
|
other_adds.append(diffitem)
|
|
elif oldtypename == 'recipe':
|
|
results = layerrecipes.filter(filepath=oldfilepath).filter(filename=oldfilename)
|
|
if len(results):
|
|
recipe = results[0]
|
|
logger.debug("Rename recipe %s to %s" % (recipe, newpath))
|
|
recipe.filepath = newfilepath
|
|
recipe.filename = newfilename
|
|
recipe.save()
|
|
update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir)
|
|
updatedrecipes.add(os.path.join(oldfilepath, oldfilename))
|
|
updatedrecipes.add(os.path.join(newfilepath, newfilename))
|
|
else:
|
|
logger.warn("Renamed recipe %s could not be found" % oldpath)
|
|
other_adds.append(diffitem)
|
|
elif oldtypename == 'bbappend':
|
|
results = layerappends.filter(filepath=oldfilepath).filter(filename=oldfilename)
|
|
if len(results):
|
|
logger.debug("Rename bbappend %s to %s" % (results[0], newfilepath))
|
|
results[0].filepath = newfilepath
|
|
results[0].filename = newfilename
|
|
results[0].save()
|
|
else:
|
|
logger.warn("Renamed bbappend %s could not be found" % oldpath)
|
|
other_adds.append(diffitem)
|
|
elif oldtypename == 'machine':
|
|
results = layermachines.filter(name=oldfilename)
|
|
if len(results):
|
|
logger.debug("Rename machine %s to %s" % (results[0], newfilename))
|
|
results[0].name = newfilename
|
|
results[0].save()
|
|
else:
|
|
logger.warn("Renamed machine %s could not be found" % oldpath)
|
|
other_adds.append(diffitem)
|
|
elif oldtypename == 'distro':
|
|
results = layerdistros.filter(name=oldfilename)
|
|
if len(results):
|
|
logger.debug("Rename distro %s to %s" % (results[0], newfilename))
|
|
results[0].name = newfilename
|
|
results[0].save()
|
|
else:
|
|
logger.warn("Renamed distro %s could not be found" % oldpath)
|
|
other_adds.append(diffitem)
|
|
elif oldtypename == 'bbclass':
|
|
results = layerclasses.filter(name=oldfilename)
|
|
if len(results):
|
|
logger.debug("Rename class %s to %s" % (results[0], newfilename))
|
|
results[0].name = newfilename
|
|
results[0].save()
|
|
else:
|
|
logger.warn("Renamed class %s could not be found" % oldpath)
|
|
other_adds.append(diffitem)
|
|
|
|
deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
|
|
for dep in deps:
|
|
dirtyrecipes.add(dep.recipe)
|
|
|
|
|
|
for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes):
|
|
path = diffitem.a_blob.path
|
|
if path.startswith(subdir_start):
|
|
skip = False
|
|
for removedir in removedirs:
|
|
if path.startswith(removedir):
|
|
skip = True
|
|
break
|
|
if skip:
|
|
continue
|
|
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
|
|
if typename == 'recipe':
|
|
values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
|
|
if len(values):
|
|
layerrecipes_delete.append(values[0])
|
|
logger.debug("Mark %s for deletion" % values[0])
|
|
updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
|
|
else:
|
|
logger.warn("Deleted recipe %s could not be found" % path)
|
|
elif typename == 'bbappend':
|
|
layerappends.filter(filepath=filepath).filter(filename=filename).delete()
|
|
elif typename == 'machine':
|
|
layermachines.filter(name=filename).delete()
|
|
elif typename == 'distro':
|
|
layerdistros.filter(name=filename).delete()
|
|
elif typename == 'bbclass':
|
|
layerclasses.filter(name=filename).delete()
|
|
|
|
for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
|
|
path = diffitem.b_blob.path
|
|
if path.startswith(subdir_start):
|
|
skip = False
|
|
for removedir in removedirs:
|
|
if path.startswith(removedir):
|
|
skip = True
|
|
break
|
|
if skip:
|
|
continue
|
|
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
|
|
if typename == 'recipe':
|
|
layerrecipes_add.append(os.path.join(repodir, path))
|
|
logger.debug("Mark %s for addition" % path)
|
|
updatedrecipes.add(os.path.join(filepath, filename))
|
|
elif typename == 'bbappend':
|
|
append = BBAppend()
|
|
append.layerbranch = layerbranch
|
|
append.filename = filename
|
|
append.filepath = filepath
|
|
append.save()
|
|
elif typename == 'machine':
|
|
machine = Machine()
|
|
machine.layerbranch = layerbranch
|
|
machine.name = filename
|
|
update_machine_conf_file(os.path.join(repodir, path), machine)
|
|
machine.save()
|
|
elif typename == 'distro':
|
|
distro = Distro()
|
|
distro.layerbranch = layerbranch
|
|
distro.name = filename
|
|
update_distro_conf_file(os.path.join(repodir, path), distro)
|
|
distro.save()
|
|
elif typename == 'bbclass':
|
|
bbclass = BBClass()
|
|
bbclass.layerbranch = layerbranch
|
|
bbclass.name = filename
|
|
bbclass.save()
|
|
|
|
for diffitem in diff.iter_change_type('M'):
|
|
path = diffitem.a_blob.path
|
|
if path.startswith(subdir_start):
|
|
skip = False
|
|
for removedir in removedirs:
|
|
if path.startswith(removedir):
|
|
skip = True
|
|
break
|
|
if skip:
|
|
continue
|
|
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
|
|
if typename == 'recipe':
|
|
logger.debug("Mark %s for update" % path)
|
|
results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
|
|
if results:
|
|
recipe = results[0]
|
|
update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
|
|
recipe.save()
|
|
updatedrecipes.add(recipe.full_path())
|
|
elif typename == 'machine':
|
|
results = layermachines.filter(name=filename)
|
|
if results:
|
|
machine = results[0]
|
|
update_machine_conf_file(os.path.join(repodir, path), machine)
|
|
machine.save()
|
|
elif typename == 'distro':
|
|
results = layerdistros.filter(name=filename)
|
|
if results:
|
|
distro = results[0]
|
|
update_distro_conf_file(os.path.join(repodir, path), distro)
|
|
distro.save()
|
|
|
|
deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
|
|
for dep in deps:
|
|
dirtyrecipes.add(dep.recipe)
|
|
|
|
for recipe in dirtyrecipes:
|
|
if not recipe.full_path() in updatedrecipes:
|
|
update_recipe_file(tinfoil, config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
|
|
else:
|
|
# Collect recipe data from scratch
|
|
|
|
layerrecipe_fns = []
|
|
if options.fullreload:
|
|
layerrecipes.delete()
|
|
else:
|
|
# First, check which recipes still exist
|
|
layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
|
|
for v in layerrecipe_values:
|
|
root = os.path.join(layerdir, v['filepath'])
|
|
fullpath = os.path.join(root, v['filename'])
|
|
preserve = True
|
|
if os.path.exists(fullpath):
|
|
for removedir in removedirs:
|
|
if fullpath.startswith(removedir):
|
|
preserve = False
|
|
break
|
|
else:
|
|
preserve = False
|
|
|
|
if preserve:
|
|
# Recipe still exists, update it
|
|
results = layerrecipes.filter(id=v['id'])[:1]
|
|
recipe = results[0]
|
|
update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir)
|
|
else:
|
|
# Recipe no longer exists, mark it for later on
|
|
layerrecipes_delete.append(v)
|
|
layerrecipe_fns.append(fullpath)
|
|
|
|
layermachines.delete()
|
|
layerdistros.delete()
|
|
layerappends.delete()
|
|
layerclasses.delete()
|
|
for root, dirs, files in os.walk(layerdir):
|
|
if '.git' in dirs:
|
|
dirs.remove('.git')
|
|
for diritem in dirs[:]:
|
|
fullpath = os.path.join(root, diritem) + os.sep
|
|
if fullpath in removedirs:
|
|
dirs.remove(diritem)
|
|
for f in files:
|
|
fullpath = os.path.join(root, f)
|
|
(typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
|
|
if typename == 'recipe':
|
|
if fullpath not in layerrecipe_fns:
|
|
layerrecipes_add.append(fullpath)
|
|
elif typename == 'bbappend':
|
|
append = BBAppend()
|
|
append.layerbranch = layerbranch
|
|
append.filename = f
|
|
append.filepath = os.path.relpath(root, layerdir)
|
|
append.save()
|
|
elif typename == 'machine':
|
|
machine = Machine()
|
|
machine.layerbranch = layerbranch
|
|
machine.name = filename
|
|
update_machine_conf_file(fullpath, machine)
|
|
machine.save()
|
|
elif typename == 'distro':
|
|
distro = Distro()
|
|
distro.layerbranch = layerbranch
|
|
distro.name = filename
|
|
update_distro_conf_file(fullpath, distro)
|
|
distro.save()
|
|
elif typename == 'bbclass':
|
|
bbclass = BBClass()
|
|
bbclass.layerbranch = layerbranch
|
|
bbclass.name = filename
|
|
bbclass.save()
|
|
|
|
for added in layerrecipes_add:
|
|
# This is good enough without actually parsing the file
|
|
(pn, pv) = split_recipe_fn(added)
|
|
oldid = -1
|
|
for deleted in layerrecipes_delete:
|
|
if deleted['pn'] == pn:
|
|
oldid = deleted['id']
|
|
layerrecipes_delete.remove(deleted)
|
|
break
|
|
if oldid > -1:
|
|
# Reclaim a record we would have deleted
|
|
results = Recipe.objects.filter(id=oldid)[:1]
|
|
recipe = results[0]
|
|
logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
|
|
else:
|
|
# Create new record
|
|
logger.debug("Add new recipe %s" % added)
|
|
recipe = Recipe()
|
|
recipe.layerbranch = layerbranch
|
|
recipe.filename = os.path.basename(added)
|
|
root = os.path.dirname(added)
|
|
recipe.filepath = os.path.relpath(root, layerdir)
|
|
update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir)
|
|
recipe.save()
|
|
|
|
for deleted in layerrecipes_delete:
|
|
logger.debug("Delete %s" % deleted)
|
|
results = Recipe.objects.filter(id=deleted['id'])[:1]
|
|
recipe = results[0]
|
|
recipe.delete()
|
|
|
|
# Save repo info
|
|
layerbranch.vcs_last_rev = topcommit.hexsha
|
|
layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
|
|
else:
|
|
logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
|
|
|
|
layerbranch.vcs_last_fetch = datetime.now()
|
|
layerbranch.save()
|
|
|
|
if options.dryrun:
|
|
raise DryRunRollbackException()
|
|
|
|
|
|
except KeyboardInterrupt:
|
|
logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
|
|
sys.exit(254)
|
|
except SystemExit:
|
|
raise
|
|
except DryRunRollbackException:
|
|
pass
|
|
except:
|
|
import traceback
|
|
traceback.print_exc()
|
|
finally:
|
|
if hasattr(tinfoil, 'shutdown'):
|
|
tinfoil.shutdown()
|
|
|
|
shutil.rmtree(tempdir)
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|