mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 20:59:01 +02:00
rrs/tools: use layer index lock
We check out different revisions while we do this processing, and so does the layer index update script, so we shouldn't be allowing both to run at once or nasty stuff will happen. Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton
parent
2afa51b108
commit
291f6bfde7
|
|
@ -71,6 +71,7 @@ def get_commit_info(info, logger):
|
|||
Recreate Maintainership history from the beginning
|
||||
"""
|
||||
def maintainer_history(options, logger):
|
||||
fetchdir = settings.LAYER_FETCH_DIR
|
||||
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
|
||||
if not maintplans.exists():
|
||||
logger.error('No enabled maintenance plans found')
|
||||
|
|
@ -78,65 +79,85 @@ def maintainer_history(options, logger):
|
|||
|
||||
no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer')
|
||||
|
||||
for maintplan in maintplans:
|
||||
for item in maintplan.maintenanceplanlayerbranch_set.all():
|
||||
layerbranch = item.layerbranch
|
||||
urldir = str(layerbranch.layer.get_fetch_dir())
|
||||
repodir = os.path.join(settings.LAYER_FETCH_DIR, urldir)
|
||||
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
||||
lockfn = os.path.join(fetchdir, "layerindex.lock")
|
||||
lockfile = utils.lock_file(lockfn)
|
||||
if not lockfile:
|
||||
logger.error("Layer index lock timeout expired")
|
||||
sys.exit(1)
|
||||
try:
|
||||
for maintplan in maintplans:
|
||||
for item in maintplan.maintenanceplanlayerbranch_set.all():
|
||||
layerbranch = item.layerbranch
|
||||
urldir = str(layerbranch.layer.get_fetch_dir())
|
||||
repodir = os.path.join(fetchdir, urldir)
|
||||
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
||||
|
||||
utils.runcmd("git checkout master -f", layerdir, logger=logger)
|
||||
maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH)
|
||||
if not os.path.exists(maintainers_full_path):
|
||||
logger.debug('No maintainers.inc for %s, skipping' % layerbranch)
|
||||
continue
|
||||
utils.runcmd("git checkout master -f", layerdir, logger=logger)
|
||||
maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH)
|
||||
if not os.path.exists(maintainers_full_path):
|
||||
logger.debug('No maintainers.inc for %s, skipping' % layerbranch)
|
||||
continue
|
||||
|
||||
commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " +
|
||||
os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger)
|
||||
commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " +
|
||||
os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger)
|
||||
|
||||
try:
|
||||
with transaction.atomic():
|
||||
for commit in commits.strip().split("\n"):
|
||||
if RecipeMaintainerHistory.objects.filter(sha1=commit):
|
||||
continue
|
||||
try:
|
||||
with transaction.atomic():
|
||||
for commit in commits.strip().split("\n"):
|
||||
if RecipeMaintainerHistory.objects.filter(sha1=commit):
|
||||
continue
|
||||
|
||||
logger.debug("Analysing commit %s ..." % (commit))
|
||||
logger.debug("Analysing commit %s ..." % (commit))
|
||||
|
||||
(author_name, author_email, date, title) = \
|
||||
get_commit_info(utils.runcmd("git show " + commit, repodir,
|
||||
logger=logger), logger)
|
||||
(author_name, author_email, date, title) = \
|
||||
get_commit_info(utils.runcmd("git show " + commit, repodir,
|
||||
logger=logger), logger)
|
||||
|
||||
author = Maintainer.create_or_update(author_name, author_email)
|
||||
rms = RecipeMaintainerHistory(title=title, date=date, author=author,
|
||||
sha1=commit)
|
||||
rms.save()
|
||||
author = Maintainer.create_or_update(author_name, author_email)
|
||||
rms = RecipeMaintainerHistory(title=title, date=date, author=author,
|
||||
sha1=commit)
|
||||
rms.save()
|
||||
|
||||
utils.runcmd("git checkout %s -f" % commit,
|
||||
repodir, logger=logger)
|
||||
utils.runcmd("git checkout %s -f" % commit,
|
||||
repodir, logger=logger)
|
||||
|
||||
lines = [line.strip() for line in open(maintainers_full_path)]
|
||||
for line in lines:
|
||||
res = get_recipe_maintainer(line, logger)
|
||||
if res:
|
||||
(pn, name, email) = res
|
||||
qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch)
|
||||
lines = [line.strip() for line in open(maintainers_full_path)]
|
||||
for line in lines:
|
||||
res = get_recipe_maintainer(line, logger)
|
||||
if res:
|
||||
(pn, name, email) = res
|
||||
qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch)
|
||||
|
||||
if qry:
|
||||
m = Maintainer.create_or_update(name, email)
|
||||
if qry:
|
||||
m = Maintainer.create_or_update(name, email)
|
||||
|
||||
rm = RecipeMaintainer()
|
||||
rm.recipe = qry[0]
|
||||
rm.maintainer = m
|
||||
rm.history = rms
|
||||
rm.save()
|
||||
|
||||
logger.debug("%s: Change maintainer to %s in commit %s." % \
|
||||
(pn, m.name, commit))
|
||||
else:
|
||||
logger.debug("%s: Not found in %s." % \
|
||||
(pn, layerbranch))
|
||||
|
||||
# set missing recipes to no maintainer
|
||||
for recipe in layerbranch.recipe_set.all():
|
||||
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
|
||||
rm = RecipeMaintainer()
|
||||
rm.recipe = qry[0]
|
||||
rm.maintainer = m
|
||||
rm.recipe = recipe
|
||||
rm.maintainer = no_maintainer
|
||||
rm.history = rms
|
||||
rm.save()
|
||||
logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \
|
||||
(recipe.pn, rms.sha1))
|
||||
|
||||
logger.debug("%s: Change maintainer to %s in commit %s." % \
|
||||
(pn, m.name, commit))
|
||||
else:
|
||||
logger.debug("%s: Not found in %s." % \
|
||||
(pn, layerbranch))
|
||||
utils.runcmd("git checkout master -f", repodir, logger=logger)
|
||||
|
||||
# set missing recipes to no maintainer
|
||||
# set new recipes to no maintainer if don't have one
|
||||
rms = RecipeMaintainerHistory.get_last()
|
||||
for recipe in layerbranch.recipe_set.all():
|
||||
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
|
||||
rm = RecipeMaintainer()
|
||||
|
|
@ -144,26 +165,14 @@ def maintainer_history(options, logger):
|
|||
rm.maintainer = no_maintainer
|
||||
rm.history = rms
|
||||
rm.save()
|
||||
logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \
|
||||
(recipe.pn, rms.sha1))
|
||||
|
||||
utils.runcmd("git checkout master -f", repodir, logger=logger)
|
||||
|
||||
# set new recipes to no maintainer if don't have one
|
||||
rms = RecipeMaintainerHistory.get_last()
|
||||
for recipe in layerbranch.recipe_set.all():
|
||||
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
|
||||
rm = RecipeMaintainer()
|
||||
rm.recipe = recipe
|
||||
rm.maintainer = no_maintainer
|
||||
rm.history = rms
|
||||
rm.save()
|
||||
logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
|
||||
(recipe.pn))
|
||||
if options.dry_run:
|
||||
raise DryRunRollbackException
|
||||
except DryRunRollbackException:
|
||||
pass
|
||||
logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
|
||||
(recipe.pn))
|
||||
if options.dry_run:
|
||||
raise DryRunRollbackException
|
||||
except DryRunRollbackException:
|
||||
pass
|
||||
finally:
|
||||
utils.unlock_file(lockfile)
|
||||
|
||||
if __name__=="__main__":
|
||||
parser = optparse.OptionParser(usage = """%prog [options]""")
|
||||
|
|
|
|||
|
|
@ -93,82 +93,91 @@ def upgrade_history(options, logger):
|
|||
if not maintplans.exists():
|
||||
logger.error('No enabled maintenance plans found')
|
||||
sys.exit(1)
|
||||
for maintplan in maintplans:
|
||||
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
|
||||
layerbranch = maintplanbranch.layerbranch
|
||||
if options.fullreload and not options.dry_run:
|
||||
RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete()
|
||||
layer = layerbranch.layer
|
||||
urldir = layer.get_fetch_dir()
|
||||
repodir = os.path.join(fetchdir, urldir)
|
||||
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
||||
|
||||
if maintplanbranch.upgrade_rev and not options.fullreload:
|
||||
initial = False
|
||||
since = maintplanbranch.upgrade_date
|
||||
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
|
||||
else:
|
||||
initial = True
|
||||
since = options.since
|
||||
since_option = '--since="%s" origin/master' % since
|
||||
lockfn = os.path.join(fetchdir, "layerindex.lock")
|
||||
lockfile = utils.lock_file(lockfn)
|
||||
if not lockfile:
|
||||
logger.error("Layer index lock timeout expired")
|
||||
sys.exit(1)
|
||||
try:
|
||||
for maintplan in maintplans:
|
||||
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
|
||||
layerbranch = maintplanbranch.layerbranch
|
||||
if options.fullreload and not options.dry_run:
|
||||
RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete()
|
||||
layer = layerbranch.layer
|
||||
urldir = layer.get_fetch_dir()
|
||||
repodir = os.path.join(fetchdir, urldir)
|
||||
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
|
||||
|
||||
repo = git.Repo(repodir)
|
||||
assert repo.bare == False
|
||||
if maintplanbranch.upgrade_rev and not options.fullreload:
|
||||
initial = False
|
||||
since = maintplanbranch.upgrade_date
|
||||
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
|
||||
else:
|
||||
initial = True
|
||||
since = options.since
|
||||
since_option = '--since="%s" origin/master' % since
|
||||
|
||||
commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option,
|
||||
repodir,
|
||||
logger=logger)
|
||||
commit_list = commits.split('\n')
|
||||
repo = git.Repo(repodir)
|
||||
assert repo.bare == False
|
||||
|
||||
bitbake_map = {}
|
||||
# Filter out some bad commits
|
||||
bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
|
||||
bitbakepath,
|
||||
logger=logger)
|
||||
bitbake_commit_list = bitbake_commits.splitlines()
|
||||
for commit in bitbake_commit_list:
|
||||
bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
|
||||
commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option,
|
||||
repodir,
|
||||
logger=logger)
|
||||
commit_list = commits.split('\n')
|
||||
|
||||
if initial:
|
||||
logger.debug("Adding initial upgrade history ....")
|
||||
bitbake_map = {}
|
||||
# Filter out some bad commits
|
||||
bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
|
||||
bitbakepath,
|
||||
logger=logger)
|
||||
bitbake_commit_list = bitbake_commits.splitlines()
|
||||
for commit in bitbake_commit_list:
|
||||
bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
|
||||
|
||||
ct, ctepoch = commit_list.pop(0).split()
|
||||
ctdate = datetime.fromtimestamp(int(ctepoch))
|
||||
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
|
||||
if initial:
|
||||
logger.debug("Adding initial upgrade history ....")
|
||||
|
||||
logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d")))
|
||||
for item in commit_list:
|
||||
if item:
|
||||
ct, ctepoch = item.split()
|
||||
ct, ctepoch = commit_list.pop(0).split()
|
||||
ctdate = datetime.fromtimestamp(int(ctepoch))
|
||||
commitobj = repo.commit(ct)
|
||||
touches_recipe = False
|
||||
for parent in commitobj.parents:
|
||||
diff = parent.diff(commitobj)
|
||||
for diffitem in diff:
|
||||
if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')):
|
||||
# We need to look at this commit
|
||||
touches_recipe = True
|
||||
break
|
||||
if touches_recipe:
|
||||
break
|
||||
if not touches_recipe:
|
||||
# No recipes changed in this commit
|
||||
# NOTE: Whilst it's possible that a change to a class might alter what's
|
||||
# in the recipe, we can ignore that since we are only concerned with actual
|
||||
# upgrades which would always require some sort of change to the recipe
|
||||
# or an include file, so we can safely skip commits that don't do that
|
||||
logger.debug("Skipping commit %s" % ct)
|
||||
continue
|
||||
logger.debug("Analysing commit %s ..." % ct)
|
||||
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
|
||||
if not options.dry_run:
|
||||
maintplanbranch.upgrade_rev = ct
|
||||
maintplanbranch.upgrade_date = ctdate
|
||||
maintplanbranch.save()
|
||||
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
|
||||
|
||||
if commit_list:
|
||||
utils.runcmd("git clean -dfx", repodir, logger=logger)
|
||||
logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d")))
|
||||
for item in commit_list:
|
||||
if item:
|
||||
ct, ctepoch = item.split()
|
||||
ctdate = datetime.fromtimestamp(int(ctepoch))
|
||||
commitobj = repo.commit(ct)
|
||||
touches_recipe = False
|
||||
for parent in commitobj.parents:
|
||||
diff = parent.diff(commitobj)
|
||||
for diffitem in diff:
|
||||
if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')):
|
||||
# We need to look at this commit
|
||||
touches_recipe = True
|
||||
break
|
||||
if touches_recipe:
|
||||
break
|
||||
if not touches_recipe:
|
||||
# No recipes changed in this commit
|
||||
# NOTE: Whilst it's possible that a change to a class might alter what's
|
||||
# in the recipe, we can ignore that since we are only concerned with actual
|
||||
# upgrades which would always require some sort of change to the recipe
|
||||
# or an include file, so we can safely skip commits that don't do that
|
||||
logger.debug("Skipping commit %s" % ct)
|
||||
continue
|
||||
logger.debug("Analysing commit %s ..." % ct)
|
||||
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
|
||||
if not options.dry_run:
|
||||
maintplanbranch.upgrade_rev = ct
|
||||
maintplanbranch.upgrade_date = ctdate
|
||||
maintplanbranch.save()
|
||||
|
||||
if commit_list:
|
||||
utils.runcmd("git clean -dfx", repodir, logger=logger)
|
||||
finally:
|
||||
utils.unlock_file(lockfile)
|
||||
|
||||
if __name__=="__main__":
|
||||
parser = optparse.OptionParser(usage = """%prog [options]""")
|
||||
|
|
|
|||
|
|
@ -147,12 +147,17 @@ if __name__=="__main__":
|
|||
|
||||
logger.debug("Starting upstream history...")
|
||||
|
||||
try:
|
||||
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
|
||||
if not maintplans.exists():
|
||||
logger.error('No enabled maintenance plans found')
|
||||
sys.exit(1)
|
||||
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
|
||||
if not maintplans.exists():
|
||||
logger.error('No enabled maintenance plans found')
|
||||
sys.exit(1)
|
||||
|
||||
lockfn = os.path.join(fetchdir, "layerindex.lock")
|
||||
lockfile = utils.lock_file(lockfn)
|
||||
if not lockfile:
|
||||
logger.error("Layer index lock timeout expired")
|
||||
sys.exit(1)
|
||||
try:
|
||||
origsyspath = sys.path
|
||||
|
||||
for maintplan in maintplans:
|
||||
|
|
@ -207,3 +212,5 @@ if __name__=="__main__":
|
|||
raise DryRunRollbackException
|
||||
except DryRunRollbackException:
|
||||
pass
|
||||
finally:
|
||||
utils.unlock_file(lockfile)
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user