From 291f6bfde72676ede6b4e58203c61df10aeff02c Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Mon, 26 Mar 2018 17:40:33 +1300 Subject: [PATCH] rrs/tools: use layer index lock We check out different revisions while we do this processing, and so does the layer index update script, so we shouldn't be allowing both to run at once or nasty stuff will happen. Signed-off-by: Paul Eggleton --- rrs/tools/rrs_maintainer_history.py | 137 +++++++++++++------------- rrs/tools/rrs_upgrade_history.py | 143 +++++++++++++++------------- rrs/tools/rrs_upstream_history.py | 17 +++- 3 files changed, 161 insertions(+), 136 deletions(-) diff --git a/rrs/tools/rrs_maintainer_history.py b/rrs/tools/rrs_maintainer_history.py index 62925fa..19559ec 100755 --- a/rrs/tools/rrs_maintainer_history.py +++ b/rrs/tools/rrs_maintainer_history.py @@ -71,6 +71,7 @@ def get_commit_info(info, logger): Recreate Maintainership history from the beginning """ def maintainer_history(options, logger): + fetchdir = settings.LAYER_FETCH_DIR maintplans = MaintenancePlan.objects.filter(updates_enabled=True) if not maintplans.exists(): logger.error('No enabled maintenance plans found') @@ -78,65 +79,85 @@ def maintainer_history(options, logger): no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer') - for maintplan in maintplans: - for item in maintplan.maintenanceplanlayerbranch_set.all(): - layerbranch = item.layerbranch - urldir = str(layerbranch.layer.get_fetch_dir()) - repodir = os.path.join(settings.LAYER_FETCH_DIR, urldir) - layerdir = os.path.join(repodir, layerbranch.vcs_subdir) + lockfn = os.path.join(fetchdir, "layerindex.lock") + lockfile = utils.lock_file(lockfn) + if not lockfile: + logger.error("Layer index lock timeout expired") + sys.exit(1) + try: + for maintplan in maintplans: + for item in maintplan.maintenanceplanlayerbranch_set.all(): + layerbranch = item.layerbranch + urldir = str(layerbranch.layer.get_fetch_dir()) + repodir = os.path.join(fetchdir, urldir) + layerdir = os.path.join(repodir, layerbranch.vcs_subdir) - utils.runcmd("git checkout master -f", layerdir, logger=logger) - maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH) - if not os.path.exists(maintainers_full_path): - logger.debug('No maintainers.inc for %s, skipping' % layerbranch) - continue + utils.runcmd("git checkout master -f", layerdir, logger=logger) + maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH) + if not os.path.exists(maintainers_full_path): + logger.debug('No maintainers.inc for %s, skipping' % layerbranch) + continue - commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " + - os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger) + commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " + + os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger) - try: - with transaction.atomic(): - for commit in commits.strip().split("\n"): - if RecipeMaintainerHistory.objects.filter(sha1=commit): - continue + try: + with transaction.atomic(): + for commit in commits.strip().split("\n"): + if RecipeMaintainerHistory.objects.filter(sha1=commit): + continue - logger.debug("Analysing commit %s ..." % (commit)) + logger.debug("Analysing commit %s ..." % (commit)) - (author_name, author_email, date, title) = \ - get_commit_info(utils.runcmd("git show " + commit, repodir, - logger=logger), logger) + (author_name, author_email, date, title) = \ + get_commit_info(utils.runcmd("git show " + commit, repodir, + logger=logger), logger) - author = Maintainer.create_or_update(author_name, author_email) - rms = RecipeMaintainerHistory(title=title, date=date, author=author, - sha1=commit) - rms.save() + author = Maintainer.create_or_update(author_name, author_email) + rms = RecipeMaintainerHistory(title=title, date=date, author=author, + sha1=commit) + rms.save() - utils.runcmd("git checkout %s -f" % commit, - repodir, logger=logger) + utils.runcmd("git checkout %s -f" % commit, + repodir, logger=logger) - lines = [line.strip() for line in open(maintainers_full_path)] - for line in lines: - res = get_recipe_maintainer(line, logger) - if res: - (pn, name, email) = res - qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) + lines = [line.strip() for line in open(maintainers_full_path)] + for line in lines: + res = get_recipe_maintainer(line, logger) + if res: + (pn, name, email) = res + qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) - if qry: - m = Maintainer.create_or_update(name, email) + if qry: + m = Maintainer.create_or_update(name, email) + rm = RecipeMaintainer() + rm.recipe = qry[0] + rm.maintainer = m + rm.history = rms + rm.save() + + logger.debug("%s: Change maintainer to %s in commit %s." % \ + (pn, m.name, commit)) + else: + logger.debug("%s: Not found in %s." % \ + (pn, layerbranch)) + + # set missing recipes to no maintainer + for recipe in layerbranch.recipe_set.all(): + if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): rm = RecipeMaintainer() - rm.recipe = qry[0] - rm.maintainer = m + rm.recipe = recipe + rm.maintainer = no_maintainer rm.history = rms rm.save() + logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \ + (recipe.pn, rms.sha1)) - logger.debug("%s: Change maintainer to %s in commit %s." % \ - (pn, m.name, commit)) - else: - logger.debug("%s: Not found in %s." % \ - (pn, layerbranch)) + utils.runcmd("git checkout master -f", repodir, logger=logger) - # set missing recipes to no maintainer + # set new recipes to no maintainer if don't have one + rms = RecipeMaintainerHistory.get_last() for recipe in layerbranch.recipe_set.all(): if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): rm = RecipeMaintainer() @@ -144,26 +165,14 @@ def maintainer_history(options, logger): rm.maintainer = no_maintainer rm.history = rms rm.save() - logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \ - (recipe.pn, rms.sha1)) - - utils.runcmd("git checkout master -f", repodir, logger=logger) - - # set new recipes to no maintainer if don't have one - rms = RecipeMaintainerHistory.get_last() - for recipe in layerbranch.recipe_set.all(): - if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): - rm = RecipeMaintainer() - rm.recipe = recipe - rm.maintainer = no_maintainer - rm.history = rms - rm.save() - logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \ - (recipe.pn)) - if options.dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass + logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \ + (recipe.pn)) + if options.dry_run: + raise DryRunRollbackException + except DryRunRollbackException: + pass + finally: + utils.unlock_file(lockfile) if __name__=="__main__": parser = optparse.OptionParser(usage = """%prog [options]""") diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py index 1c37bea..a03b5be 100755 --- a/rrs/tools/rrs_upgrade_history.py +++ b/rrs/tools/rrs_upgrade_history.py @@ -93,82 +93,91 @@ def upgrade_history(options, logger): if not maintplans.exists(): logger.error('No enabled maintenance plans found') sys.exit(1) - for maintplan in maintplans: - for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(): - layerbranch = maintplanbranch.layerbranch - if options.fullreload and not options.dry_run: - RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete() - layer = layerbranch.layer - urldir = layer.get_fetch_dir() - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, layerbranch.vcs_subdir) - if maintplanbranch.upgrade_rev and not options.fullreload: - initial = False - since = maintplanbranch.upgrade_date - since_option = '%s..origin/master' % maintplanbranch.upgrade_rev - else: - initial = True - since = options.since - since_option = '--since="%s" origin/master' % since + lockfn = os.path.join(fetchdir, "layerindex.lock") + lockfile = utils.lock_file(lockfn) + if not lockfile: + logger.error("Layer index lock timeout expired") + sys.exit(1) + try: + for maintplan in maintplans: + for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(): + layerbranch = maintplanbranch.layerbranch + if options.fullreload and not options.dry_run: + RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete() + layer = layerbranch.layer + urldir = layer.get_fetch_dir() + repodir = os.path.join(fetchdir, urldir) + layerdir = os.path.join(repodir, layerbranch.vcs_subdir) - repo = git.Repo(repodir) - assert repo.bare == False + if maintplanbranch.upgrade_rev and not options.fullreload: + initial = False + since = maintplanbranch.upgrade_date + since_option = '%s..origin/master' % maintplanbranch.upgrade_rev + else: + initial = True + since = options.since + since_option = '--since="%s" origin/master' % since - commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option, - repodir, - logger=logger) - commit_list = commits.split('\n') + repo = git.Repo(repodir) + assert repo.bare == False - bitbake_map = {} - # Filter out some bad commits - bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", - bitbakepath, - logger=logger) - bitbake_commit_list = bitbake_commits.splitlines() - for commit in bitbake_commit_list: - bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' + commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option, + repodir, + logger=logger) + commit_list = commits.split('\n') - if initial: - logger.debug("Adding initial upgrade history ....") + bitbake_map = {} + # Filter out some bad commits + bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", + bitbakepath, + logger=logger) + bitbake_commit_list = bitbake_commits.splitlines() + for commit in bitbake_commit_list: + bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' - ct, ctepoch = commit_list.pop(0).split() - ctdate = datetime.fromtimestamp(int(ctepoch)) - run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) + if initial: + logger.debug("Adding initial upgrade history ....") - logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d"))) - for item in commit_list: - if item: - ct, ctepoch = item.split() + ct, ctepoch = commit_list.pop(0).split() ctdate = datetime.fromtimestamp(int(ctepoch)) - commitobj = repo.commit(ct) - touches_recipe = False - for parent in commitobj.parents: - diff = parent.diff(commitobj) - for diffitem in diff: - if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')): - # We need to look at this commit - touches_recipe = True - break - if touches_recipe: - break - if not touches_recipe: - # No recipes changed in this commit - # NOTE: Whilst it's possible that a change to a class might alter what's - # in the recipe, we can ignore that since we are only concerned with actual - # upgrades which would always require some sort of change to the recipe - # or an include file, so we can safely skip commits that don't do that - logger.debug("Skipping commit %s" % ct) - continue - logger.debug("Analysing commit %s ..." % ct) - run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) - if not options.dry_run: - maintplanbranch.upgrade_rev = ct - maintplanbranch.upgrade_date = ctdate - maintplanbranch.save() + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) - if commit_list: - utils.runcmd("git clean -dfx", repodir, logger=logger) + logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d"))) + for item in commit_list: + if item: + ct, ctepoch = item.split() + ctdate = datetime.fromtimestamp(int(ctepoch)) + commitobj = repo.commit(ct) + touches_recipe = False + for parent in commitobj.parents: + diff = parent.diff(commitobj) + for diffitem in diff: + if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')): + # We need to look at this commit + touches_recipe = True + break + if touches_recipe: + break + if not touches_recipe: + # No recipes changed in this commit + # NOTE: Whilst it's possible that a change to a class might alter what's + # in the recipe, we can ignore that since we are only concerned with actual + # upgrades which would always require some sort of change to the recipe + # or an include file, so we can safely skip commits that don't do that + logger.debug("Skipping commit %s" % ct) + continue + logger.debug("Analysing commit %s ..." % ct) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) + if not options.dry_run: + maintplanbranch.upgrade_rev = ct + maintplanbranch.upgrade_date = ctdate + maintplanbranch.save() + + if commit_list: + utils.runcmd("git clean -dfx", repodir, logger=logger) + finally: + utils.unlock_file(lockfile) if __name__=="__main__": parser = optparse.OptionParser(usage = """%prog [options]""") diff --git a/rrs/tools/rrs_upstream_history.py b/rrs/tools/rrs_upstream_history.py index 5e0590f..a3bf798 100755 --- a/rrs/tools/rrs_upstream_history.py +++ b/rrs/tools/rrs_upstream_history.py @@ -147,12 +147,17 @@ if __name__=="__main__": logger.debug("Starting upstream history...") - try: - maintplans = MaintenancePlan.objects.filter(updates_enabled=True) - if not maintplans.exists(): - logger.error('No enabled maintenance plans found') - sys.exit(1) + maintplans = MaintenancePlan.objects.filter(updates_enabled=True) + if not maintplans.exists(): + logger.error('No enabled maintenance plans found') + sys.exit(1) + lockfn = os.path.join(fetchdir, "layerindex.lock") + lockfile = utils.lock_file(lockfn) + if not lockfile: + logger.error("Layer index lock timeout expired") + sys.exit(1) + try: origsyspath = sys.path for maintplan in maintplans: @@ -207,3 +212,5 @@ if __name__=="__main__": raise DryRunRollbackException except DryRunRollbackException: pass + finally: + utils.unlock_file(lockfile)