mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-05 05:04:46 +02:00
4712f00df7
File "/opt/layerindex/layerindex/update.py", line 525
failed_layers[branch].append('%s: Failed to add since LAYERDEPENDS [%s ...] is not
SyntaxError: '(' was never closed
Signed-off-by: Tim Orling <tim.orling@konsulko.com>
610 lines
28 KiB
Python
Executable File
610 lines
28 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Fetch layer repositories and update layer index database
|
|
#
|
|
# Copyright (C) 2013-2016 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
#
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
|
|
import sys
|
|
import os
|
|
import optparse
|
|
import codecs
|
|
import logging
|
|
import subprocess
|
|
from datetime import datetime, timedelta
|
|
from pkg_resources import parse_version
|
|
import utils
|
|
import operator
|
|
import re
|
|
import multiprocessing
|
|
|
|
import warnings
|
|
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
|
|
|
logger = utils.logger_create('LayerIndexUpdate')
|
|
|
|
# Ensure PythonGit is installed (buildhistory_analysis needs it)
|
|
try:
|
|
import git
|
|
except ImportError:
|
|
logger.error("Please install PythonGit 0.3.1 or later in order to use this script")
|
|
sys.exit(1)
|
|
|
|
|
|
def prepare_update_layer_command(options, branch, layer, initial=False):
|
|
"""Prepare the update_layer.py command line"""
|
|
if branch.update_environment:
|
|
cmdprefix = branch.update_environment.get_command()
|
|
else:
|
|
cmdprefix = 'python3'
|
|
cmd = '%s update_layer.py -l %s -b %s' % (cmdprefix, layer.name, branch.name)
|
|
if options.reload:
|
|
cmd += ' --reload'
|
|
if options.fullreload:
|
|
cmd += ' --fullreload'
|
|
if options.nocheckout:
|
|
cmd += ' --nocheckout'
|
|
if options.dryrun:
|
|
cmd += ' -n'
|
|
if initial:
|
|
cmd += ' -i'
|
|
if options.loglevel == logging.DEBUG:
|
|
cmd += ' -d'
|
|
elif options.loglevel == logging.ERROR:
|
|
cmd += ' -q'
|
|
if options.keep_temp:
|
|
cmd += ' --keep-temp'
|
|
if options.stop_on_error:
|
|
cmd += ' --stop-on-error'
|
|
return cmd
|
|
|
|
def update_actual_branch(layerquery, fetchdir, branch, options, update_bitbake, bitbakepath):
|
|
"""Update actual branch for layers and bitbake in database"""
|
|
to_save = set()
|
|
actual_branch = options.actual_branch
|
|
if update_bitbake:
|
|
branchobj = utils.get_branch(branch)
|
|
if actual_branch != branchobj.bitbake_branch:
|
|
if utils.is_branch_valid(bitbakepath, actual_branch):
|
|
logger.info("bitbake: %s.bitbake_branch: %s -> %s" % (branch, branchobj.bitbake_branch, actual_branch))
|
|
branchobj.bitbake_branch = actual_branch
|
|
to_save.add(branchobj)
|
|
else:
|
|
logger.info("Skipping update bitbake_branch for bitbake - branch %s doesn't exist" % actual_branch)
|
|
else:
|
|
logger.info("bitbake: %s.bitbake_branch is already %s, so no change" % (branch, actual_branch))
|
|
|
|
for layer in layerquery:
|
|
urldir = layer.get_fetch_dir()
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
if not utils.is_branch_valid(repodir, actual_branch):
|
|
logger.info("Skipping update actual_branch for %s - branch %s doesn't exist" % (layer.name, actual_branch))
|
|
continue
|
|
layerbranch = layer.get_layerbranch(branch)
|
|
if not layerbranch:
|
|
logger.info("Skipping update actual_branch for %s - layerbranch %s doesn't exist" % (layer.name, branch))
|
|
continue
|
|
if actual_branch != layerbranch.actual_branch:
|
|
logger.info("%s: %s.actual_branch: %s -> %s" % (layer.name, branch, layerbranch.actual_branch, actual_branch))
|
|
layerbranch.actual_branch = actual_branch
|
|
to_save.add(layerbranch)
|
|
else:
|
|
logger.info("%s: %s.actual_branch is already %s, so no change" % (layer.name, branch, actual_branch))
|
|
|
|
# At last, do the save
|
|
if not options.dryrun:
|
|
for s in to_save:
|
|
s.save()
|
|
|
|
def fetch_repo(vcs_url, repodir, urldir, fetchdir, layer_name):
|
|
logger.info("Fetching remote repository %s" % vcs_url)
|
|
try:
|
|
if not os.path.exists(repodir):
|
|
utils.runcmd(['git', 'clone', vcs_url, urldir], fetchdir, logger=logger, printerr=False)
|
|
else:
|
|
utils.runcmd(['git', 'fetch', '-p'], repodir, logger=logger, printerr=False)
|
|
return (vcs_url, None)
|
|
except subprocess.CalledProcessError as e:
|
|
logger.error("Fetch of layer %s failed: %s" % (layer_name, e.output))
|
|
return (vcs_url, e.output)
|
|
|
|
def print_subdir_error(newbranch, layername, vcs_subdir, branchdesc):
|
|
# This will error out if the directory is completely invalid or had never existed at this point
|
|
# If it previously existed but has since been deleted, you will get the revision where it was
|
|
# deleted - so we need to handle that case separately later
|
|
if newbranch:
|
|
logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layername, branchdesc, vcs_subdir))
|
|
elif vcs_subdir:
|
|
logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layername, branchdesc))
|
|
|
|
def extract_value(valuename, output):
|
|
res = re.search("^%s = \"(.*)\"" % valuename, output, re.M)
|
|
if res:
|
|
return res.group(1) or ''
|
|
else:
|
|
return ''
|
|
|
|
def main():
|
|
if parse_version(git.__version__) < parse_version('0.3.1'):
|
|
logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
|
|
sys.exit(1)
|
|
|
|
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options]""")
|
|
|
|
parser.add_option("-b", "--branch",
|
|
help = "Specify branch(es) to update (use commas to separate multiple). Default is all enabled branches.",
|
|
action="store", dest="branch", default='')
|
|
parser.add_option("-l", "--layer",
|
|
help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
|
|
action="store", dest="layers")
|
|
parser.add_option("-t", "--timeout",
|
|
help = "Specify timeout in seconds to get layerindex.lock. Default is 30 seconds.",
|
|
type="int", action="store", dest="timeout", default=30)
|
|
parser.add_option("-r", "--reload",
|
|
help = "Reload recipe data instead of updating since last update",
|
|
action="store_true", dest="reload")
|
|
parser.add_option("", "--fullreload",
|
|
help = "Discard existing recipe data and fetch it from scratch",
|
|
action="store_true", dest="fullreload")
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("-x", "--nofetch",
|
|
help = "Don't fetch repositories",
|
|
action="store_true", dest="nofetch")
|
|
parser.add_option("", "--nocheckout",
|
|
help = "Don't check out branches",
|
|
action="store_true", dest="nocheckout")
|
|
parser.add_option("", "--stop-on-error",
|
|
help = "Stop on first parsing error",
|
|
action="store_true", default=False, dest="stop_on_error")
|
|
parser.add_option("-a", "--actual-branch",
|
|
help = "Update actual branch for layer and bitbake",
|
|
action="store", dest="actual_branch", default='')
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
parser.add_option("", "--keep-temp",
|
|
help = "Preserve temporary directory at the end instead of deleting it",
|
|
action="store_true")
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
if len(args) > 1:
|
|
logger.error('unexpected argument "%s"' % args[1])
|
|
parser.print_help()
|
|
sys.exit(1)
|
|
|
|
utils.setup_django()
|
|
import settings
|
|
from layerindex.models import Branch, LayerItem, Update, LayerUpdate, LayerBranch
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
if options.branch:
|
|
branches = options.branch.split(',')
|
|
for branch in branches:
|
|
if not utils.get_branch(branch):
|
|
logger.error("Specified branch %s is not valid" % branch)
|
|
sys.exit(1)
|
|
else:
|
|
branchquery = Branch.objects.filter(updates_enabled=True)
|
|
branches = [branch.name for branch in branchquery]
|
|
|
|
fetchdir = settings.LAYER_FETCH_DIR
|
|
if not fetchdir:
|
|
logger.error("Please set LAYER_FETCH_DIR in settings.py")
|
|
sys.exit(1)
|
|
|
|
|
|
# We deliberately exclude status == 'X' ("no update") here
|
|
layerquery_all = LayerItem.objects.filter(comparison=False).filter(status='P')
|
|
if layerquery_all.count() == 0:
|
|
logger.info("No published layers to update")
|
|
sys.exit(1)
|
|
|
|
# For -a option to update bitbake branch
|
|
update_bitbake = False
|
|
if options.layers:
|
|
layers = options.layers.split(',')
|
|
if 'bitbake' in layers:
|
|
update_bitbake = True
|
|
layers.remove('bitbake')
|
|
for layer in layers:
|
|
layerquery = LayerItem.objects.filter(comparison=False).filter(name=layer)
|
|
if layerquery.count() == 0:
|
|
logger.error('No layers matching specified query "%s"' % layer)
|
|
sys.exit(1)
|
|
layerquery = LayerItem.objects.filter(comparison=False).filter(name__in=layers)
|
|
else:
|
|
layerquery = layerquery_all
|
|
update_bitbake = True
|
|
|
|
if options.actual_branch:
|
|
if not options.branch:
|
|
logger.error("-a option requires -b")
|
|
sys.exit(1)
|
|
elif len(branches) != 1:
|
|
logger.error("Only one branch should be used with -a")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(fetchdir):
|
|
os.makedirs(fetchdir)
|
|
|
|
allrepos = {}
|
|
fetchedresult = []
|
|
fetchedrepos = []
|
|
failedrepos = {}
|
|
|
|
# We don't want git to prompt for any passwords (e.g. when accessing renamed/hidden github repos)
|
|
os.environ['SSH_ASKPASS'] = ''
|
|
os.environ['GIT_ASKPASS'] = ''
|
|
os.environ['GIT_TERMINAL_PROMPT'] = '0'
|
|
|
|
listhandler = utils.ListHandler()
|
|
listhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
|
logger.addHandler(listhandler)
|
|
|
|
update = Update()
|
|
update.started = datetime.now()
|
|
if options.fullreload or options.reload:
|
|
update.reload = True
|
|
else:
|
|
update.reload = False
|
|
if not options.dryrun:
|
|
update.save()
|
|
try:
|
|
lockfn = os.path.join(fetchdir, "layerindex.lock")
|
|
lockfile = utils.lock_file(lockfn, options.timeout, logger)
|
|
if not lockfile:
|
|
logger.error("Layer index lock timeout expired")
|
|
sys.exit(1)
|
|
try:
|
|
# Make sure oe-core is fetched since recipe parsing requires it
|
|
layerquery_core = LayerItem.objects.filter(comparison=False).filter(name=settings.CORE_LAYER_NAME)
|
|
if layerquery_core in layerquery:
|
|
layerquery_fetch = list(layerquery)
|
|
else:
|
|
layerquery_fetch = list(layerquery) + list(layerquery_core)
|
|
# Fetch latest metadata from repositories
|
|
for layer in layerquery_fetch:
|
|
# Handle multiple layers in a single repo
|
|
urldir = layer.get_fetch_dir()
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
if layer.vcs_url not in allrepos:
|
|
allrepos[layer.vcs_url] = (repodir, urldir, fetchdir, layer.name)
|
|
# Add bitbake
|
|
if settings.BITBAKE_REPO_URL not in allrepos:
|
|
bitbakeitem = LayerItem()
|
|
bitbakeitem.vcs_url = settings.BITBAKE_REPO_URL
|
|
bitbakeurldir = bitbakeitem.get_fetch_dir()
|
|
bitbakepath = os.path.join(fetchdir, bitbakeurldir)
|
|
allrepos[settings.BITBAKE_REPO_URL] = (bitbakepath, bitbakeurldir, fetchdir, "bitbake")
|
|
|
|
(bitbakepath, _, _, _) = allrepos[settings.BITBAKE_REPO_URL]
|
|
if getattr(settings, 'BITBAKE_PATH', ''):
|
|
bitbakepath = os.path.join(bitbakepath, settings.BITBAKE_PATH)
|
|
|
|
if not options.nofetch:
|
|
# Parallel fetching
|
|
pool = multiprocessing.Pool(int(settings.PARALLEL_JOBS))
|
|
for url in allrepos:
|
|
fetchedresult.append(pool.apply_async(fetch_repo, \
|
|
(url, allrepos[url][0], allrepos[url][1], allrepos[url][2], allrepos[url][3],)))
|
|
pool.close()
|
|
pool.join()
|
|
|
|
for url in fetchedresult[:]:
|
|
# The format is (url, error), the error is None when succeed.
|
|
if url.get()[1]:
|
|
failedrepos[url.get()[0]] = url.get()[1]
|
|
else:
|
|
fetchedrepos.append(url.get()[0])
|
|
|
|
if not (fetchedrepos or update_bitbake):
|
|
logger.error("No repositories could be fetched, exiting")
|
|
sys.exit(1)
|
|
|
|
if options.actual_branch:
|
|
update_actual_branch(layerquery, fetchdir, branches[0], options, update_bitbake, bitbakepath)
|
|
return
|
|
|
|
# Get a safe bitbake branch to call into from this script (used later on)
|
|
safe_bitbake_branch = 'origin/master'
|
|
master_branch = Branch.objects.filter(name='master').first()
|
|
if master_branch and master_branch.bitbake_branch:
|
|
safe_bitbake_branch = 'origin/' + master_branch.bitbake_branch
|
|
|
|
# Process and extract data from each layer
|
|
# We now do this by calling out to a separate script; doing otherwise turned out to be
|
|
# unreliable due to leaking memory (we're using bitbake internals in a manner in which
|
|
# they never get used during normal operation).
|
|
failed_layers = {}
|
|
for branch in branches:
|
|
failed_layers[branch] = []
|
|
# If layer_A depends(or recommends) on layer_B, add layer_B before layer_A
|
|
deps_dict_all = {}
|
|
layerquery_sorted = []
|
|
collections = set()
|
|
branchobj = utils.get_branch(branch)
|
|
for layer in layerquery_all:
|
|
# Get all collections from database, but we can't trust the
|
|
# one which will be updated since its collections maybe
|
|
# changed (different from database).
|
|
if layer in layerquery:
|
|
continue
|
|
layerbranch = layer.get_layerbranch(branch)
|
|
if layerbranch:
|
|
collections.add((layerbranch.collection, layerbranch.version))
|
|
|
|
for layer in layerquery:
|
|
if layer.vcs_url in failedrepos:
|
|
logger.info("Skipping update of layer %s - fetch failed" % layer.name)
|
|
continue
|
|
|
|
layerbranch = layer.get_layerbranch(branch)
|
|
branchname = branch
|
|
branchdesc = branch
|
|
newbranch = False
|
|
branchobj = utils.get_branch(branch)
|
|
if layerbranch:
|
|
if layerbranch.actual_branch:
|
|
branchname = layerbranch.actual_branch
|
|
branchdesc = "%s (%s)" % (branch, branchname)
|
|
|
|
layerbranch_updates_enabled = LayerBranch.objects.filter(layer=layer,
|
|
branch=branchobj.id, updates_enabled=True)
|
|
if not layerbranch_updates_enabled:
|
|
logger.info("Skipping update of layer %s branch %s - updates disabled" % (layer.name, branchname))
|
|
continue
|
|
else:
|
|
# LayerBranch doesn't exist for this branch, create it temporarily
|
|
# (we won't save this - update_layer.py will do the actual creation
|
|
# if it gets called).
|
|
newbranch = True
|
|
layerbranch = LayerBranch()
|
|
layerbranch.layer = layer
|
|
layerbranch.branch = branchobj
|
|
layerbranch_source = layer.get_layerbranch(branchobj)
|
|
if not layerbranch_source:
|
|
layerbranch_source = layer.get_layerbranch(None)
|
|
if layerbranch_source:
|
|
layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
|
|
|
|
# Collect repo info
|
|
urldir = layer.get_fetch_dir()
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
repo = git.Repo(repodir)
|
|
if repo.bare:
|
|
logger.error('Repository %s is bare, not supported' % repodir)
|
|
continue
|
|
try:
|
|
# Always get origin/branchname, so it raises error when branch doesn't exist when nocheckout
|
|
topcommit = repo.commit('origin/%s' % branchname)
|
|
if options.nocheckout:
|
|
topcommit = repo.commit('HEAD')
|
|
except:
|
|
if newbranch:
|
|
logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
|
|
else:
|
|
logger.info("layer %s - branch %s no longer exists, removing it from database" % (layer.name, branchdesc))
|
|
if not options.dryrun:
|
|
layerbranch.delete()
|
|
continue
|
|
|
|
if layerbranch.vcs_subdir and not options.nocheckout:
|
|
# Find latest commit in subdirectory
|
|
# A bit odd to do it this way but apparently there's no other way in the GitPython API
|
|
topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
|
|
if not topcommit:
|
|
print_subdir_error(newbranch, layer.name, layerbranch.vcs_subdir, branchdesc)
|
|
if not (newbranch and layerbranch.vcs_subdir):
|
|
logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
|
|
continue
|
|
|
|
if layerbranch.vcs_last_rev == topcommit.hexsha and not update.reload:
|
|
logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
|
|
collections.add((layerbranch.collection, layerbranch.version))
|
|
continue
|
|
else:
|
|
# Check out appropriate branch
|
|
if not options.nocheckout:
|
|
utils.checkout_layer_branch(layerbranch, repodir, logger=logger)
|
|
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
|
if layerbranch.vcs_subdir and not os.path.exists(layerdir):
|
|
print_subdir_error(newbranch, layer.name, layerbranch.vcs_subdir, branchdesc)
|
|
continue
|
|
|
|
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
|
|
logger.error("conf/layer.conf not found for layer %s (branch %s) - is subdirectory set correctly?" % (layer.name, branch))
|
|
continue
|
|
|
|
cmd = prepare_update_layer_command(options, branchobj, layer, initial=True)
|
|
logger.debug('Running layer update command: %s' % cmd)
|
|
ret, output = utils.run_command_interruptible(cmd)
|
|
logger.debug('output: %s' % output)
|
|
if ret == 254:
|
|
# Interrupted by user, break out of loop
|
|
logger.info('Update interrupted, exiting')
|
|
sys.exit(254)
|
|
elif ret != 0:
|
|
output = output.rstrip()
|
|
# Save a layerupdate here or we won't see this output
|
|
layerupdate = LayerUpdate()
|
|
layerupdate.update = update
|
|
layerupdate.layer = layer
|
|
layerupdate.branch = branchobj
|
|
layerupdate.started = datetime.now()
|
|
layerupdate.log = output
|
|
layerupdate.retcode = ret
|
|
if not options.dryrun:
|
|
layerupdate.save()
|
|
continue
|
|
|
|
col = extract_value('BBFILE_COLLECTIONS', output)
|
|
if not col:
|
|
logger.error('Unable to find BBFILE_COLLECTIONS value in initial output')
|
|
# Assume (perhaps naively) that it's an error specific to the layer
|
|
continue
|
|
ver = extract_value('LAYERVERSION', output)
|
|
deps = extract_value('LAYERDEPENDS', output)
|
|
recs = extract_value('LAYERRECOMMENDS', output)
|
|
|
|
if not options.nocheckout:
|
|
# We need to check this out because we're using stuff from bb.utils
|
|
# below, and if we don't it might be a python 2 revision which would
|
|
# be an issue
|
|
utils.checkout_repo(bitbakepath, safe_bitbake_branch, logger=logger)
|
|
|
|
deps_dict = utils.explode_dep_versions2(bitbakepath, deps)
|
|
recs_dict = utils.explode_dep_versions2(bitbakepath, recs)
|
|
if not (deps_dict or recs_dict):
|
|
# No depends, add it firstly
|
|
layerquery_sorted.append(layer)
|
|
collections.add((col, ver))
|
|
continue
|
|
deps_dict_all[layer] = {'deps': deps_dict, \
|
|
'recs': recs_dict, \
|
|
'collection': col, \
|
|
'version': ver}
|
|
|
|
# Move deps_dict_all to layerquery_sorted orderly
|
|
if deps_dict_all:
|
|
logger.info("Sorting layers for branch %s" % branch)
|
|
while True:
|
|
deps_dict_all_copy = deps_dict_all.copy()
|
|
for layer, value in deps_dict_all_copy.items():
|
|
for deps_recs in ('deps', 'recs'):
|
|
for req_col, req_ver_list in value[deps_recs].copy().items():
|
|
matched = False
|
|
if req_ver_list:
|
|
req_ver = req_ver_list[0]
|
|
else:
|
|
req_ver = None
|
|
if utils.is_deps_satisfied(req_col, req_ver, collections):
|
|
del(value[deps_recs][req_col])
|
|
if not (value['deps'] or value['recs']):
|
|
# All the depends are in collections:
|
|
del(deps_dict_all[layer])
|
|
layerquery_sorted.append(layer)
|
|
collections.add((value['collection'], value['version']))
|
|
|
|
if not len(deps_dict_all):
|
|
break
|
|
|
|
finished = True
|
|
# If nothing changed after a run, drop recs and try again
|
|
if operator.eq(deps_dict_all_copy, deps_dict_all):
|
|
for layer, value in deps_dict_all.items():
|
|
if value['recs'] and not value['deps']:
|
|
# Add it if recs isn't satisfied only.
|
|
logger.warn('Adding %s without LAYERRECOMMENDS...' % layer.name)
|
|
del(deps_dict_all[layer])
|
|
layerquery_sorted.append(layer)
|
|
collections.add((value['collection'], value['version']))
|
|
failed_msg = '%s: Added without LAYERRECOMMENDS' % layer.name
|
|
failed_layers[branch].append(failed_msg)
|
|
finished = False
|
|
break
|
|
if not finished:
|
|
continue
|
|
logger.warning("Cannot find required collections on branch %s:" % branch)
|
|
for layer, value in deps_dict_all.items():
|
|
logger.warn('%s: LAYERDEPENDS: %s LAYERRECOMMENDS: %s' % (layer.name, value['deps'], value['recs']))
|
|
if value['deps']:
|
|
failed_layers[branch].append('%s: Failed to add since LAYERDEPENDS [%s ...] is not satisfied' % (layer.name, next(iter(value['deps']))))
|
|
else:
|
|
# Should never come here
|
|
logger.error("Unexpected errors when sorting layers")
|
|
sys.exit(1)
|
|
logger.warning("Known collections on branch %s: %s" % (branch, collections))
|
|
break
|
|
|
|
for layer in layerquery_sorted:
|
|
layerupdate = LayerUpdate()
|
|
layerupdate.update = update
|
|
layerupdate.layer = layer
|
|
layerupdate.branch = branchobj
|
|
layerbranch = layer.get_layerbranch(branch)
|
|
if layerbranch:
|
|
layerupdate.vcs_before_rev = layerbranch.vcs_last_rev
|
|
|
|
errmsg = failedrepos.get(layer.vcs_url, '')
|
|
if errmsg:
|
|
logger.info("Skipping update of layer %s as fetch of repository %s failed:\n%s" % (layer.name, layer.vcs_url, errmsg))
|
|
layerupdate.started = datetime.now()
|
|
layerupdate.finished = datetime.now()
|
|
layerupdate.log = 'ERROR: fetch failed: %s' % errmsg
|
|
if not options.dryrun:
|
|
layerupdate.save()
|
|
continue
|
|
|
|
layerupdate.started = datetime.now()
|
|
if not options.dryrun:
|
|
layerupdate.save()
|
|
cmd = prepare_update_layer_command(options, branchobj, layer)
|
|
logger.debug('Running layer update command: %s' % cmd)
|
|
ret, output = utils.run_command_interruptible(cmd)
|
|
layerupdate.finished = datetime.now()
|
|
|
|
# We need to get layerbranch here because it might not have existed until
|
|
# layer_update.py created it, but it still may not create one (e.g. if subdir
|
|
# didn't exist) so we still need to check
|
|
layerbranch = layer.get_layerbranch(branch)
|
|
if layerbranch:
|
|
layerupdate.vcs_after_rev = layerbranch.vcs_last_rev
|
|
layerupdate.log = output
|
|
layerupdate.retcode = ret
|
|
if not options.dryrun:
|
|
layerupdate.save()
|
|
|
|
if ret == 254:
|
|
# Interrupted by user, break out of loop
|
|
logger.info('Update interrupted, exiting')
|
|
sys.exit(254)
|
|
if options.stop_on_error and ret != 0:
|
|
logger.info('Layer update failed with --stop-on-error, stopping')
|
|
sys.exit(1)
|
|
if failed_layers:
|
|
for branch, err_msg_list in failed_layers.items():
|
|
if err_msg_list:
|
|
print()
|
|
logger.error("Issues found on branch %s:\n %s" % (branch, "\n ".join(err_msg_list)))
|
|
print()
|
|
finally:
|
|
utils.unlock_file(lockfile)
|
|
|
|
except KeyboardInterrupt:
|
|
logger.info('Update interrupted, exiting')
|
|
sys.exit(254)
|
|
except Exception:
|
|
import traceback
|
|
logger.error(traceback.format_exc().rstrip())
|
|
sys.exit(1)
|
|
finally:
|
|
update.log = ''.join(listhandler.read())
|
|
update.finished = datetime.now()
|
|
if not options.dryrun:
|
|
update.save()
|
|
|
|
if not options.dryrun:
|
|
# Purge old update records
|
|
update_purge_days = getattr(settings, 'UPDATE_PURGE_DAYS', 30)
|
|
Update.objects.filter(started__lte=datetime.now()-timedelta(days=update_purge_days)).delete()
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|