MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-19 20:59:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

rrs/tools: use layer index lock

Browse Source 
We check out different revisions while we do this processing, and so
does the layer index update script, so we shouldn't be allowing both to
run at once or nasty stuff will happen.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2018-03-26 17:40:33 +13:00
parent 2afa51b108
commit 291f6bfde7
3 changed files with 161 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
rrs/tools/rrs_maintainer_history.py
View File

@ -71,6 +71,7 @@ def get_commit_info(info, logger):
Recreate Maintainership history from the beginning Recreate Maintainership history from the beginning
""" """
def maintainer_history(options, logger): def maintainer_history(options, logger):
fetchdir = settings.LAYER_FETCH_DIR
maintplans = MaintenancePlan.objects.filter(updates_enabled=True) maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists(): if not maintplans.exists():
logger.error('No enabled maintenance plans found') logger.error('No enabled maintenance plans found')
@ -78,65 +79,85 @@ def maintainer_history(options, logger):
no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer') no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer')
for maintplan in maintplans: lockfn = os.path.join(fetchdir, "layerindex.lock")
for item in maintplan.maintenanceplanlayerbranch_set.all(): lockfile = utils.lock_file(lockfn)
layerbranch = item.layerbranch if not lockfile:
urldir = str(layerbranch.layer.get_fetch_dir()) logger.error("Layer index lock timeout expired")
repodir = os.path.join(settings.LAYER_FETCH_DIR, urldir) sys.exit(1)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir) try:
for maintplan in maintplans:
for item in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = item.layerbranch
urldir = str(layerbranch.layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
utils.runcmd("git checkout master -f", layerdir, logger=logger) utils.runcmd("git checkout master -f", layerdir, logger=logger)
maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH) maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH)
if not os.path.exists(maintainers_full_path): if not os.path.exists(maintainers_full_path):
logger.debug('No maintainers.inc for %s, skipping' % layerbranch) logger.debug('No maintainers.inc for %s, skipping' % layerbranch)
continue continue
commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " + commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " +
os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger) os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger)
try: try:
with transaction.atomic(): with transaction.atomic():
for commit in commits.strip().split("\n"): for commit in commits.strip().split("\n"):
if RecipeMaintainerHistory.objects.filter(sha1=commit): if RecipeMaintainerHistory.objects.filter(sha1=commit):
continue continue
logger.debug("Analysing commit %s ..." % (commit)) logger.debug("Analysing commit %s ..." % (commit))
(author_name, author_email, date, title) = \ (author_name, author_email, date, title) = \
get_commit_info(utils.runcmd("git show " + commit, repodir, get_commit_info(utils.runcmd("git show " + commit, repodir,
logger=logger), logger) logger=logger), logger)
author = Maintainer.create_or_update(author_name, author_email) author = Maintainer.create_or_update(author_name, author_email)
rms = RecipeMaintainerHistory(title=title, date=date, author=author, rms = RecipeMaintainerHistory(title=title, date=date, author=author,
sha1=commit) sha1=commit)
rms.save() rms.save()
utils.runcmd("git checkout %s -f" % commit, utils.runcmd("git checkout %s -f" % commit,
repodir, logger=logger) repodir, logger=logger)
lines = [line.strip() for line in open(maintainers_full_path)] lines = [line.strip() for line in open(maintainers_full_path)]
for line in lines: for line in lines:
res = get_recipe_maintainer(line, logger) res = get_recipe_maintainer(line, logger)
if res: if res:
(pn, name, email) = res (pn, name, email) = res
qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch)
if qry: if qry:
m = Maintainer.create_or_update(name, email) m = Maintainer.create_or_update(name, email)
rm = RecipeMaintainer()
rm.recipe = qry[0]
rm.maintainer = m
rm.history = rms
rm.save()
logger.debug("%s: Change maintainer to %s in commit %s." % \
(pn, m.name, commit))
else:
logger.debug("%s: Not found in %s." % \
(pn, layerbranch))
# set missing recipes to no maintainer
for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
rm = RecipeMaintainer() rm = RecipeMaintainer()
rm.recipe = qry[0] rm.recipe = recipe
rm.maintainer = m rm.maintainer = no_maintainer
rm.history = rms rm.history = rms
rm.save() rm.save()
logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \
(recipe.pn, rms.sha1))
logger.debug("%s: Change maintainer to %s in commit %s." % \ utils.runcmd("git checkout master -f", repodir, logger=logger)
(pn, m.name, commit))
else:
logger.debug("%s: Not found in %s." % \
(pn, layerbranch))
# set missing recipes to no maintainer # set new recipes to no maintainer if don't have one
rms = RecipeMaintainerHistory.get_last()
for recipe in layerbranch.recipe_set.all(): for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
rm = RecipeMaintainer() rm = RecipeMaintainer()
@ -144,26 +165,14 @@ def maintainer_history(options, logger):
rm.maintainer = no_maintainer rm.maintainer = no_maintainer
rm.history = rms rm.history = rms
rm.save() rm.save()
logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \ logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
(recipe.pn, rms.sha1)) (recipe.pn))
if options.dry_run:
utils.runcmd("git checkout master -f", repodir, logger=logger) raise DryRunRollbackException
except DryRunRollbackException:
# set new recipes to no maintainer if don't have one pass
rms = RecipeMaintainerHistory.get_last() finally:
for recipe in layerbranch.recipe_set.all(): utils.unlock_file(lockfile)
if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms):
rm = RecipeMaintainer()
rm.recipe = recipe
rm.maintainer = no_maintainer
rm.history = rms
rm.save()
logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
(recipe.pn))
if options.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
if __name__=="__main__": if __name__=="__main__":
parser = optparse.OptionParser(usage = """%prog [options]""") parser = optparse.OptionParser(usage = """%prog [options]""")

143
rrs/tools/rrs_upgrade_history.py
View File

@ -93,82 +93,91 @@ def upgrade_history(options, logger):
if not maintplans.exists(): if not maintplans.exists():
logger.error('No enabled maintenance plans found') logger.error('No enabled maintenance plans found')
sys.exit(1) sys.exit(1)
for maintplan in maintplans:
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if maintplanbranch.upgrade_rev and not options.fullreload: lockfn = os.path.join(fetchdir, "layerindex.lock")
initial = False lockfile = utils.lock_file(lockfn)
since = maintplanbranch.upgrade_date if not lockfile:
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev logger.error("Layer index lock timeout expired")
else: sys.exit(1)
initial = True try:
since = options.since for maintplan in maintplans:
since_option = '--since="%s" origin/master' % since for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
repo = git.Repo(repodir) if maintplanbranch.upgrade_rev and not options.fullreload:
assert repo.bare == False initial = False
since = maintplanbranch.upgrade_date
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
else:
initial = True
since = options.since
since_option = '--since="%s" origin/master' % since
commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option, repo = git.Repo(repodir)
repodir, assert repo.bare == False
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {} commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option,
# Filter out some bad commits repodir,
bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", logger=logger)
bitbakepath, commit_list = commits.split('\n')
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
if initial: bitbake_map = {}
logger.debug("Adding initial upgrade history ....") # Filter out some bad commits
bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
ct, ctepoch = commit_list.pop(0).split() if initial:
ctdate = datetime.fromtimestamp(int(ctepoch)) logger.debug("Adding initial upgrade history ....")
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d"))) ct, ctepoch = commit_list.pop(0).split()
for item in commit_list:
if item:
ct, ctepoch = item.split()
ctdate = datetime.fromtimestamp(int(ctepoch)) ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct) run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
if not options.dry_run:
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
if commit_list: logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d")))
utils.runcmd("git clean -dfx", repodir, logger=logger) for item in commit_list:
if item:
ct, ctepoch = item.split()
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
if not options.dry_run:
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
if commit_list:
utils.runcmd("git clean -dfx", repodir, logger=logger)
finally:
utils.unlock_file(lockfile)
if __name__=="__main__": if __name__=="__main__":
parser = optparse.OptionParser(usage = """%prog [options]""") parser = optparse.OptionParser(usage = """%prog [options]""")

17
rrs/tools/rrs_upstream_history.py
View File

@ -147,12 +147,17 @@ if __name__=="__main__":
logger.debug("Starting upstream history...") logger.debug("Starting upstream history...")
try: maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
maintplans = MaintenancePlan.objects.filter(updates_enabled=True) if not maintplans.exists():
if not maintplans.exists(): logger.error('No enabled maintenance plans found')
logger.error('No enabled maintenance plans found') sys.exit(1)
sys.exit(1)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
origsyspath = sys.path origsyspath = sys.path
for maintplan in maintplans: for maintplan in maintplans:
@ -207,3 +212,5 @@ if __name__=="__main__":
raise DryRunRollbackException raise DryRunRollbackException
except DryRunRollbackException: except DryRunRollbackException:
pass pass
finally:
utils.unlock_file(lockfile)