diff --git a/rrs/tools/rrs_maintainer_history.py b/rrs/tools/rrs_maintainer_history.py index 62925fa..19559ec 100755 --- a/rrs/tools/rrs_maintainer_history.py +++ b/rrs/tools/rrs_maintainer_history.py @@ -71,6 +71,7 @@ def get_commit_info(info, logger): Recreate Maintainership history from the beginning """ def maintainer_history(options, logger): + fetchdir = settings.LAYER_FETCH_DIR maintplans = MaintenancePlan.objects.filter(updates_enabled=True) if not maintplans.exists(): logger.error('No enabled maintenance plans found') @@ -78,65 +79,85 @@ def maintainer_history(options, logger): no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer') - for maintplan in maintplans: - for item in maintplan.maintenanceplanlayerbranch_set.all(): - layerbranch = item.layerbranch - urldir = str(layerbranch.layer.get_fetch_dir()) - repodir = os.path.join(settings.LAYER_FETCH_DIR, urldir) - layerdir = os.path.join(repodir, layerbranch.vcs_subdir) + lockfn = os.path.join(fetchdir, "layerindex.lock") + lockfile = utils.lock_file(lockfn) + if not lockfile: + logger.error("Layer index lock timeout expired") + sys.exit(1) + try: + for maintplan in maintplans: + for item in maintplan.maintenanceplanlayerbranch_set.all(): + layerbranch = item.layerbranch + urldir = str(layerbranch.layer.get_fetch_dir()) + repodir = os.path.join(fetchdir, urldir) + layerdir = os.path.join(repodir, layerbranch.vcs_subdir) - utils.runcmd("git checkout master -f", layerdir, logger=logger) - maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH) - if not os.path.exists(maintainers_full_path): - logger.debug('No maintainers.inc for %s, skipping' % layerbranch) - continue + utils.runcmd("git checkout master -f", layerdir, logger=logger) + maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH) + if not os.path.exists(maintainers_full_path): + logger.debug('No maintainers.inc for %s, skipping' % layerbranch) + continue - commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " + - os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger) + commits = utils.runcmd("git log --format='%H' --reverse --date=rfc " + + os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH), repodir, logger=logger) - try: - with transaction.atomic(): - for commit in commits.strip().split("\n"): - if RecipeMaintainerHistory.objects.filter(sha1=commit): - continue + try: + with transaction.atomic(): + for commit in commits.strip().split("\n"): + if RecipeMaintainerHistory.objects.filter(sha1=commit): + continue - logger.debug("Analysing commit %s ..." % (commit)) + logger.debug("Analysing commit %s ..." % (commit)) - (author_name, author_email, date, title) = \ - get_commit_info(utils.runcmd("git show " + commit, repodir, - logger=logger), logger) + (author_name, author_email, date, title) = \ + get_commit_info(utils.runcmd("git show " + commit, repodir, + logger=logger), logger) - author = Maintainer.create_or_update(author_name, author_email) - rms = RecipeMaintainerHistory(title=title, date=date, author=author, - sha1=commit) - rms.save() + author = Maintainer.create_or_update(author_name, author_email) + rms = RecipeMaintainerHistory(title=title, date=date, author=author, + sha1=commit) + rms.save() - utils.runcmd("git checkout %s -f" % commit, - repodir, logger=logger) + utils.runcmd("git checkout %s -f" % commit, + repodir, logger=logger) - lines = [line.strip() for line in open(maintainers_full_path)] - for line in lines: - res = get_recipe_maintainer(line, logger) - if res: - (pn, name, email) = res - qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) + lines = [line.strip() for line in open(maintainers_full_path)] + for line in lines: + res = get_recipe_maintainer(line, logger) + if res: + (pn, name, email) = res + qry = Recipe.objects.filter(pn = pn, layerbranch = layerbranch) - if qry: - m = Maintainer.create_or_update(name, email) + if qry: + m = Maintainer.create_or_update(name, email) + rm = RecipeMaintainer() + rm.recipe = qry[0] + rm.maintainer = m + rm.history = rms + rm.save() + + logger.debug("%s: Change maintainer to %s in commit %s." % \ + (pn, m.name, commit)) + else: + logger.debug("%s: Not found in %s." % \ + (pn, layerbranch)) + + # set missing recipes to no maintainer + for recipe in layerbranch.recipe_set.all(): + if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): rm = RecipeMaintainer() - rm.recipe = qry[0] - rm.maintainer = m + rm.recipe = recipe + rm.maintainer = no_maintainer rm.history = rms rm.save() + logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \ + (recipe.pn, rms.sha1)) - logger.debug("%s: Change maintainer to %s in commit %s." % \ - (pn, m.name, commit)) - else: - logger.debug("%s: Not found in %s." % \ - (pn, layerbranch)) + utils.runcmd("git checkout master -f", repodir, logger=logger) - # set missing recipes to no maintainer + # set new recipes to no maintainer if don't have one + rms = RecipeMaintainerHistory.get_last() for recipe in layerbranch.recipe_set.all(): if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): rm = RecipeMaintainer() @@ -144,26 +165,14 @@ def maintainer_history(options, logger): rm.maintainer = no_maintainer rm.history = rms rm.save() - logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \ - (recipe.pn, rms.sha1)) - - utils.runcmd("git checkout master -f", repodir, logger=logger) - - # set new recipes to no maintainer if don't have one - rms = RecipeMaintainerHistory.get_last() - for recipe in layerbranch.recipe_set.all(): - if not RecipeMaintainer.objects.filter(recipe = recipe, history = rms): - rm = RecipeMaintainer() - rm.recipe = recipe - rm.maintainer = no_maintainer - rm.history = rms - rm.save() - logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \ - (recipe.pn)) - if options.dry_run: - raise DryRunRollbackException - except DryRunRollbackException: - pass + logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \ + (recipe.pn)) + if options.dry_run: + raise DryRunRollbackException + except DryRunRollbackException: + pass + finally: + utils.unlock_file(lockfile) if __name__=="__main__": parser = optparse.OptionParser(usage = """%prog [options]""") diff --git a/rrs/tools/rrs_upgrade_history.py b/rrs/tools/rrs_upgrade_history.py index 1c37bea..a03b5be 100755 --- a/rrs/tools/rrs_upgrade_history.py +++ b/rrs/tools/rrs_upgrade_history.py @@ -93,82 +93,91 @@ def upgrade_history(options, logger): if not maintplans.exists(): logger.error('No enabled maintenance plans found') sys.exit(1) - for maintplan in maintplans: - for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(): - layerbranch = maintplanbranch.layerbranch - if options.fullreload and not options.dry_run: - RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete() - layer = layerbranch.layer - urldir = layer.get_fetch_dir() - repodir = os.path.join(fetchdir, urldir) - layerdir = os.path.join(repodir, layerbranch.vcs_subdir) - if maintplanbranch.upgrade_rev and not options.fullreload: - initial = False - since = maintplanbranch.upgrade_date - since_option = '%s..origin/master' % maintplanbranch.upgrade_rev - else: - initial = True - since = options.since - since_option = '--since="%s" origin/master' % since + lockfn = os.path.join(fetchdir, "layerindex.lock") + lockfile = utils.lock_file(lockfn) + if not lockfile: + logger.error("Layer index lock timeout expired") + sys.exit(1) + try: + for maintplan in maintplans: + for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(): + layerbranch = maintplanbranch.layerbranch + if options.fullreload and not options.dry_run: + RecipeUpgrade.objects.filter(recipe__layerbranch=layerbranch).delete() + layer = layerbranch.layer + urldir = layer.get_fetch_dir() + repodir = os.path.join(fetchdir, urldir) + layerdir = os.path.join(repodir, layerbranch.vcs_subdir) - repo = git.Repo(repodir) - assert repo.bare == False + if maintplanbranch.upgrade_rev and not options.fullreload: + initial = False + since = maintplanbranch.upgrade_date + since_option = '%s..origin/master' % maintplanbranch.upgrade_rev + else: + initial = True + since = options.since + since_option = '--since="%s" origin/master' % since - commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option, - repodir, - logger=logger) - commit_list = commits.split('\n') + repo = git.Repo(repodir) + assert repo.bare == False - bitbake_map = {} - # Filter out some bad commits - bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", - bitbakepath, - logger=logger) - bitbake_commit_list = bitbake_commits.splitlines() - for commit in bitbake_commit_list: - bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' + commits = utils.runcmd("git log %s --format='%%H %%ct' --reverse" % since_option, + repodir, + logger=logger) + commit_list = commits.split('\n') - if initial: - logger.debug("Adding initial upgrade history ....") + bitbake_map = {} + # Filter out some bad commits + bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^", + bitbakepath, + logger=logger) + bitbake_commit_list = bitbake_commits.splitlines() + for commit in bitbake_commit_list: + bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d' - ct, ctepoch = commit_list.pop(0).split() - ctdate = datetime.fromtimestamp(int(ctepoch)) - run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) + if initial: + logger.debug("Adding initial upgrade history ....") - logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d"))) - for item in commit_list: - if item: - ct, ctepoch = item.split() + ct, ctepoch = commit_list.pop(0).split() ctdate = datetime.fromtimestamp(int(ctepoch)) - commitobj = repo.commit(ct) - touches_recipe = False - for parent in commitobj.parents: - diff = parent.diff(commitobj) - for diffitem in diff: - if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')): - # We need to look at this commit - touches_recipe = True - break - if touches_recipe: - break - if not touches_recipe: - # No recipes changed in this commit - # NOTE: Whilst it's possible that a change to a class might alter what's - # in the recipe, we can ignore that since we are only concerned with actual - # upgrades which would always require some sort of change to the recipe - # or an include file, so we can safely skip commits that don't do that - logger.debug("Skipping commit %s" % ct) - continue - logger.debug("Analysing commit %s ..." % ct) - run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) - if not options.dry_run: - maintplanbranch.upgrade_rev = ct - maintplanbranch.upgrade_date = ctdate - maintplanbranch.save() + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True) - if commit_list: - utils.runcmd("git clean -dfx", repodir, logger=logger) + logger.debug("Adding upgrade history from %s to %s ..." % (since, datetime.today().strftime("%Y-%m-%d"))) + for item in commit_list: + if item: + ct, ctepoch = item.split() + ctdate = datetime.fromtimestamp(int(ctepoch)) + commitobj = repo.commit(ct) + touches_recipe = False + for parent in commitobj.parents: + diff = parent.diff(commitobj) + for diffitem in diff: + if diffitem.a_path.endswith(('.bb', '.inc')) or diffitem.b_path.endswith(('.bb', '.inc')): + # We need to look at this commit + touches_recipe = True + break + if touches_recipe: + break + if not touches_recipe: + # No recipes changed in this commit + # NOTE: Whilst it's possible that a change to a class might alter what's + # in the recipe, we can ignore that since we are only concerned with actual + # upgrades which would always require some sort of change to the recipe + # or an include file, so we can safely skip commits that don't do that + logger.debug("Skipping commit %s" % ct) + continue + logger.debug("Analysing commit %s ..." % ct) + run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map) + if not options.dry_run: + maintplanbranch.upgrade_rev = ct + maintplanbranch.upgrade_date = ctdate + maintplanbranch.save() + + if commit_list: + utils.runcmd("git clean -dfx", repodir, logger=logger) + finally: + utils.unlock_file(lockfile) if __name__=="__main__": parser = optparse.OptionParser(usage = """%prog [options]""") diff --git a/rrs/tools/rrs_upstream_history.py b/rrs/tools/rrs_upstream_history.py index 5e0590f..a3bf798 100755 --- a/rrs/tools/rrs_upstream_history.py +++ b/rrs/tools/rrs_upstream_history.py @@ -147,12 +147,17 @@ if __name__=="__main__": logger.debug("Starting upstream history...") - try: - maintplans = MaintenancePlan.objects.filter(updates_enabled=True) - if not maintplans.exists(): - logger.error('No enabled maintenance plans found') - sys.exit(1) + maintplans = MaintenancePlan.objects.filter(updates_enabled=True) + if not maintplans.exists(): + logger.error('No enabled maintenance plans found') + sys.exit(1) + lockfn = os.path.join(fetchdir, "layerindex.lock") + lockfile = utils.lock_file(lockfn) + if not lockfile: + logger.error("Layer index lock timeout expired") + sys.exit(1) + try: origsyspath = sys.path for maintplan in maintplans: @@ -207,3 +212,5 @@ if __name__=="__main__": raise DryRunRollbackException except DryRunRollbackException: pass + finally: + utils.unlock_file(lockfile)