rrs_upgrade_history.py: enable to work across python2/3 versions

If you want to go back and get history for the earlier releases (krogoth
and previous) then we need to be able to support both python 2 and 3,
which practically means we need the same split for this script as we
have for the main layer index update script.

The catch here is that since we are going back and following the history
of changes forward, we basically need to use the same version of bitbake
that was current at that time. This works except for around the
transition between python 2 to 3 where the metadata lagged behind a bit,
so we need to take that into account. In order to keep things generic we
have a date field on the maintenance plan layer branch that specifies
the date in the metadata where we should switch over to python 3, and
then link to PythonEnvironment records that should be used for python 2
and 3 respectively.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2018-03-23 01:30:56 +13:00
parent 59b66d5505
commit ba27f9fee0
4 changed files with 356 additions and 184 deletions

View File

@ -0,0 +1,31 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import datetime
class Migration(migrations.Migration):
dependencies = [
('layerindex', '0010_add_dependencies'),
('rrs', '0006_maintplan_email'),
]
operations = [
migrations.AddField(
model_name='maintenanceplanlayerbranch',
name='python2_environment',
field=models.ForeignKey(blank=True, null=True, help_text='Environment to use for Python 2 commits', related_name='maintplan_layerbranch_python2_set', to='layerindex.PythonEnvironment'),
),
migrations.AddField(
model_name='maintenanceplanlayerbranch',
name='python3_environment',
field=models.ForeignKey(blank=True, null=True, help_text='Environment to use for Python 3 commits', related_name='maintplan_layerbranch_python3_set', to='layerindex.PythonEnvironment'),
),
migrations.AddField(
model_name='maintenanceplanlayerbranch',
name='python3_switch_date',
field=models.DateTimeField(verbose_name='Commit date to switch to Python 3', default=datetime.datetime(2016, 6, 2, 0, 0)),
),
]

View File

@ -9,11 +9,11 @@ import os
import os.path import os.path
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../'))) sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../')))
from datetime import date from datetime import date, datetime
from django.db import models from django.db import models
from django.contrib.auth.models import User from django.contrib.auth.models import User
from layerindex.models import Recipe, LayerBranch from layerindex.models import Recipe, LayerBranch, PythonEnvironment
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
@ -36,6 +36,9 @@ class MaintenancePlan(models.Model):
class MaintenancePlanLayerBranch(models.Model): class MaintenancePlanLayerBranch(models.Model):
plan = models.ForeignKey(MaintenancePlan) plan = models.ForeignKey(MaintenancePlan)
layerbranch = models.ForeignKey(LayerBranch) layerbranch = models.ForeignKey(LayerBranch)
python3_switch_date = models.DateTimeField('Commit date to switch to Python 3', default=datetime(2016, 6, 2))
python2_environment = models.ForeignKey(PythonEnvironment, related_name='maintplan_layerbranch_python2_set', blank=True, null=True, help_text='Environment to use for Python 2 commits')
python3_environment = models.ForeignKey(PythonEnvironment, related_name='maintplan_layerbranch_python3_set', blank=True, null=True, help_text='Environment to use for Python 3 commits')
class Meta: class Meta:
verbose_name_plural = "Maintenance plan layer branches" verbose_name_plural = "Maintenance plan layer branches"

View File

@ -4,13 +4,13 @@
# #
# To detect package versions of the recipes the script uses the name of the recipe. # To detect package versions of the recipes the script uses the name of the recipe.
# #
# Copyright (C) 2015 Intel Corporation # Copyright (C) 2015, 2018 Intel Corporation
# Author: Anibal Limon <anibal.limon@linux.intel.com> # Authors: Anibal Limon <anibal.limon@linux.intel.com>
# Paul Eggleton <paul.eggleton@linux.intel.com>
# #
# Licensed under the MIT license, see COPYING.MIT for details # Licensed under the MIT license, see COPYING.MIT for details
from datetime import datetime from datetime import datetime, timedelta
from datetime import timedelta
import sys import sys
import os.path import os.path
@ -18,199 +18,73 @@ import optparse
import logging import logging
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__)))) sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__))))
from common import common_setup, get_pv_type, load_recipes, \ from common import common_setup, get_logger
get_logger, DryRunRollbackException
common_setup() common_setup()
from layerindex import utils, recipeparse from layerindex import utils
from layerindex.update_layer import split_recipe_fn
utils.setup_django() utils.setup_django()
from django.db import transaction
import settings import settings
logger = get_logger("HistoryUpgrade", settings) logger = get_logger("HistoryUpgrade", settings)
fetchdir = settings.LAYER_FETCH_DIR fetchdir = settings.LAYER_FETCH_DIR
bitbakepath = os.path.join(fetchdir, 'bitbake')
if not fetchdir: if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py") logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1) sys.exit(1)
# setup bitbake
bitbakepath = os.path.join(fetchdir, 'bitbake')
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
from bb.utils import vercmp_string
""" def run_internal(maintplanlayerbranch, commit, commitdate, options, logger, bitbake_map, initial=False):
Store upgrade into RecipeUpgrade model. from layerindex.models import PythonEnvironment
""" from rrs.models import Release
def _save_upgrade(recipe, pv, commit, title, info, logger): if commitdate < maintplanlayerbranch.python3_switch_date:
from email.utils import parsedate_tz, mktime_tz # Python 2
from rrs.models import Maintainer, RecipeUpgrade if maintplanlayerbranch.python2_environment:
cmdprefix = maintplanlayerbranch.python2_environment.get_command()
maintainer_name = info.split(';')[0] else:
maintainer_email = info.split(';')[1] cmdprefix = 'python'
author_date = info.split(';')[2] # Ensure we're using a bitbake version that is python 2 compatible
commit_date = info.split(';')[3] if commitdate > datetime(2016, 5, 10):
commitdate = datetime(2016, 5, 10)
maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email)
upgrade = RecipeUpgrade()
upgrade.recipe = recipe
upgrade.maintainer = maintainer
upgrade.author_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(author_date)))
upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(commit_date)))
upgrade.version = pv
upgrade.sha1 = commit
upgrade.title = title.strip()
upgrade.save()
"""
Create upgrade receives new recipe_data and cmp versions.
"""
def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False):
from layerindex.models import Recipe
from rrs.models import RecipeUpgrade
pn = recipe_data.getVar('PN', True)
pv = recipe_data.getVar('PV', True)
try:
recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch)
except Exception as e:
logger.warn("%s: Not found in Layer branch %s." %
(pn, str(layerbranch)))
return
try:
latest_upgrade = RecipeUpgrade.objects.filter(
recipe = recipe).order_by('-commit_date')[0]
prev_pv = latest_upgrade.version
except:
prev_pv = None
if prev_pv is None:
logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv))
_save_upgrade(recipe, pv, ct, title, info, logger)
else: else:
from common import get_recipe_pv_without_srcpv # Python 3
if maintplanlayerbranch.python3_environment:
cmdprefix = maintplanlayerbranch.python3_environment.get_command()
else:
cmdprefix = 'python3'
(ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv, bitbake_rev = utils.runcmd('git rev-list -1 --before="%s" origin/master' % str(commitdate),
get_pv_type(prev_pv)) bitbakepath, logger=logger)
(npv, _, _) = get_recipe_pv_without_srcpv(pv, check_rev = bitbake_map.get(bitbake_rev, None)
get_pv_type(pv)) if check_rev:
logger.debug('Preferring bitbake revision %s over %s' % (check_rev, bitbake_rev))
try: bitbake_rev = check_rev
if npv == 'git':
logger.debug("%s: Avoiding upgrade to unversioned git." % \
(recipe.pn))
elif ppv == 'git' or vercmp_string(ppv, npv) == -1:
if initial is True:
logger.debug("%s: Update initial upgrade ( -> %s)." % \
(recipe.pn, pv))
latest_upgrade.version = pv
latest_upgrade.save()
else:
logger.debug("%s: detected upgrade (%s -> %s)" \
" in ct %s." % (pn, prev_pv, pv, ct))
_save_upgrade(recipe, pv, ct, title, info, logger)
except:
logger.error("%s: fail to detect upgrade (%s -> %s)" \
" in ct %s." % (pn, prev_pv, pv, ct))
"""
Returns a list containing the fullpaths to the recipes from a commit.
"""
def _get_recipes_filenames(ct, repodir, layerdir, logger):
ct_files = []
layerdir_start = os.path.normpath(layerdir) + os.sep
files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct,
repodir, logger=logger)
for f in files.split("\n"):
if f != "":
fullpath = os.path.join(repodir, f)
# Skip deleted files in commit
if not os.path.exists(fullpath):
continue
(typename, _, filename) = recipeparse.detect_file_type(fullpath,
layerdir_start)
if typename == 'recipe':
ct_files.append(fullpath)
return ct_files
def do_initial(layerbranch, ct, logger, dry_run):
layer = layerbranch.layer
urldir = str(layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
utils.runcmd("git checkout %s" % ct,
repodir, logger=logger)
utils.runcmd("git clean -dfx", repodir, logger=logger)
title = "Initial import at 1.6 release start."
info = "No maintainer;;Mon, 11 Nov 2013 00:00:00 +0000;Mon, 11 Nov 2013 00:00:00 +0000"
(tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath,
fetchdir, settings, logger, nocheckout=True)
try:
with transaction.atomic():
for recipe_data in recipes:
_create_upgrade(recipe_data, layerbranch, '', title,
info, logger, initial=True)
if dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
tinfoil.shutdown()
def do_loop(layerbranch, ct, logger, dry_run):
layer = layerbranch.layer
urldir = str(layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
utils.runcmd("git checkout %s" % ct,
repodir, logger=logger)
utils.runcmd("git clean -dfx", repodir, logger=logger)
fns = _get_recipes_filenames(ct, repodir, layerdir, logger)
if not fns:
return
(tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath,
fetchdir, settings, logger, recipe_files=fns,
nocheckout=True)
title = utils.runcmd("git log --format='%s' -n 1 " + ct,
repodir, logger=logger)
info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \
+ ct, destdir=repodir, logger=logger)
try:
with transaction.atomic():
for recipe_data in recipes:
_create_upgrade(recipe_data, layerbranch, ct, title,
info, logger)
if dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
tinfoil.shutdown()
cmd = '%s upgrade_history_internal.py %s %s' % (cmdprefix, maintplanlayerbranch.layerbranch.id, commit)
if initial:
release = Release.get_by_date(maintplanlayerbranch.plan, commitdate)
if release:
comment = 'Initial import at %s release start.' % release.name
else:
comment = 'Initial import at %s' % commit
cmd += ' --initial="%s"' % comment
if bitbake_rev:
cmd += ' --bitbake-rev %s' % bitbake_rev
if options.dry_run:
cmd += ' --dry-run'
if options.loglevel == logging.DEBUG:
cmd += ' --debug'
logger.debug('Running %s' % cmd)
ret, output = utils.run_command_interruptible(cmd)
if ret == 254:
# Interrupted by user, break out of loop
logger.info('Update interrupted, exiting')
sys.exit(254)
""" """
Upgrade history handler. Upgrade history handler.
""" """
def upgrade_history(options, logger): def upgrade_history(options, logger):
from layerindex.models import LayerBranch
from rrs.models import MaintenancePlan from rrs.models import MaintenancePlan
# start date # start date
@ -221,6 +95,7 @@ def upgrade_history(options, logger):
since = "2013-11-11" since = "2013-11-11"
#RecipeUpgrade.objects.all().delete() #RecipeUpgrade.objects.all().delete()
else: else:
# FIXME this is awful - we should be storing the last commit somewhere
since = (now - timedelta(days=8)).strftime("%Y-%m-%d") since = (now - timedelta(days=8)).strftime("%Y-%m-%d")
maintplans = MaintenancePlan.objects.filter(updates_enabled=True) maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
@ -236,21 +111,33 @@ def upgrade_history(options, logger):
layerdir = os.path.join(repodir, layerbranch.vcs_subdir) layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
commits = utils.runcmd("git log --since='" + since + commits = utils.runcmd("git log --since='" + since +
"' --format='%H' --reverse", repodir, "' --format='%H %ct' --reverse origin/master", repodir,
logger=logger) logger=logger)
commit_list = commits.split('\n') commit_list = commits.split('\n')
bitbake_map = {}
# Filter out some bad commits
bitbake_commits = utils.runcmd("git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
if options.initial: if options.initial:
logger.debug("Adding initial upgrade history ....") logger.debug("Adding initial upgrade history ....")
ct = commit_list.pop(0) ct, ctepoch = commit_list.pop(0).split()
do_initial(layerbranch, ct, logger, options.dry_run) ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map, initial=True)
logger.debug("Adding upgrade history from %s to %s ..." % (since, today)) logger.debug("Adding upgrade history from %s to %s ..." % (since, today))
for ct in commit_list: for item in commit_list:
if ct: if item:
ct, ctepoch = item.split()
ctdate = datetime.fromtimestamp(int(ctepoch))
logger.debug("Analysing commit %s ..." % ct) logger.debug("Analysing commit %s ..." % ct)
do_loop(layerbranch, ct, logger, options.dry_run) run_internal(maintplanbranch, ct, ctdate, options, logger, bitbake_map)
if commit_list: if commit_list:
utils.runcmd("git clean -dfx", repodir, logger=logger) utils.runcmd("git clean -dfx", repodir, logger=logger)

View File

@ -0,0 +1,251 @@
# Internal script called by rrs_upgrade_history.py
#
# To detect package versions of the recipes the script uses the name of the recipe.
#
# Copyright (C) 2015, 2018 Intel Corporation
# Authors: Anibal Limon <anibal.limon@linux.intel.com>
# Paul Eggleton <paul.eggleton@linux.intel.com>
#
# Licensed under the MIT license, see COPYING.MIT for details
from datetime import datetime
import sys
import os
import optparse
import logging
import re
from distutils.version import LooseVersion
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__))))
from common import common_setup, get_pv_type, load_recipes, \
get_logger, DryRunRollbackException
common_setup()
from layerindex import utils, recipeparse
from layerindex.update_layer import split_recipe_fn
"""
Store upgrade into RecipeUpgrade model.
"""
def _save_upgrade(recipe, pv, commit, title, info, logger):
from email.utils import parsedate_tz, mktime_tz
from rrs.models import Maintainer, RecipeUpgrade
maintainer_name = info.split(';')[0]
maintainer_email = info.split(';')[1]
author_date = info.split(';')[2]
commit_date = info.split(';')[3]
maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email)
upgrade = RecipeUpgrade()
upgrade.recipe = recipe
upgrade.maintainer = maintainer
upgrade.author_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(author_date)))
upgrade.commit_date = datetime.utcfromtimestamp(mktime_tz(
parsedate_tz(commit_date)))
upgrade.version = pv
upgrade.sha1 = commit
upgrade.title = title.strip()
upgrade.save()
"""
Create upgrade receives new recipe_data and cmp versions.
"""
def _create_upgrade(recipe_data, layerbranch, ct, title, info, logger, initial=False):
from layerindex.models import Recipe
from rrs.models import RecipeUpgrade
from bb.utils import vercmp_string
pn = recipe_data.getVar('PN', True)
pv = recipe_data.getVar('PV', True)
try:
recipe = Recipe.objects.get(pn=pn, layerbranch=layerbranch)
except Exception as e:
logger.warn("%s: Not found in Layer branch %s." %
(pn, str(layerbranch)))
return
try:
latest_upgrade = RecipeUpgrade.objects.filter(
recipe = recipe).order_by('-commit_date')[0]
prev_pv = latest_upgrade.version
except KeyboardInterrupt:
raise
except:
prev_pv = None
if prev_pv is None:
logger.debug("%s: Initial upgrade ( -> %s)." % (recipe.pn, pv))
_save_upgrade(recipe, pv, ct, title, info, logger)
else:
from common import get_recipe_pv_without_srcpv
(ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv,
get_pv_type(prev_pv))
(npv, _, _) = get_recipe_pv_without_srcpv(pv,
get_pv_type(pv))
try:
if npv == 'git':
logger.debug("%s: Avoiding upgrade to unversioned git." % \
(recipe.pn))
elif ppv == 'git' or vercmp_string(ppv, npv) == -1:
if initial is True:
logger.debug("%s: Update initial upgrade ( -> %s)." % \
(recipe.pn, pv))
latest_upgrade.version = pv
latest_upgrade.save()
else:
logger.debug("%s: detected upgrade (%s -> %s)" \
" in ct %s." % (pn, prev_pv, pv, ct))
_save_upgrade(recipe, pv, ct, title, info, logger)
except KeyboardInterrupt:
raise
except:
logger.error("%s: fail to detect upgrade (%s -> %s)" \
" in ct %s." % (pn, prev_pv, pv, ct))
"""
Returns a list containing the fullpaths to the recipes from a commit.
"""
def _get_recipes_filenames(ct, repodir, layerdir, logger):
ct_files = []
layerdir_start = os.path.normpath(layerdir) + os.sep
files = utils.runcmd("git log --name-only --format='%n' -n 1 " + ct,
repodir, logger=logger)
for f in files.split("\n"):
if f != "":
fullpath = os.path.join(repodir, f)
# Skip deleted files in commit
if not os.path.exists(fullpath):
continue
(typename, _, filename) = recipeparse.detect_file_type(fullpath,
layerdir_start)
if typename == 'recipe':
ct_files.append(fullpath)
return ct_files
def generate_history(options, layerbranch_id, commit, logger):
from layerindex.models import LayerBranch
from rrs.models import Release
layerbranch = LayerBranch.objects.get(id=layerbranch_id)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
layer = layerbranch.layer
urldir = str(layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
utils.runcmd("git checkout %s" % commit,
repodir, logger=logger)
utils.runcmd("git clean -dfx", repodir, logger=logger)
if options.initial:
fns = None
else:
fns = _get_recipes_filenames(commit, repodir, layerdir, logger)
if not fns:
return
# setup bitbake
bitbakepath = os.path.join(fetchdir, 'bitbake')
if options.bitbake_rev:
bitbake_rev = options.bitbake_rev
if not re.match('^[0-9a-f]{40}$', bitbake_rev):
# Branch name, need to check out detached
bitbake_rev = 'origin/%s' % bitbake_rev
else:
commitdate = utils.runcmd("git show -s --format=%ci", repodir, logger=logger)
bitbake_rev = '`git rev-list -1 --before="%s" origin/master`' % commitdate
utils.runcmd('git checkout %s' % bitbake_rev,
bitbakepath, logger=logger)
utils.runcmd("git clean -dfx", bitbakepath, logger=logger)
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
(tinfoil, d, recipes) = load_recipes(layerbranch, bitbakepath,
fetchdir, settings, logger, recipe_files=fns,
nocheckout=True)
if options.initial:
title = options.initial
info = 'No maintainer;;' + utils.runcmd("git log --format='%ad;%cd' --date=rfc -n 1 " \
+ commit, destdir=repodir, logger=logger)
recordcommit = ''
else:
title = utils.runcmd("git log --format='%s' -n 1 " + commit,
repodir, logger=logger)
info = utils.runcmd("git log --format='%an;%ae;%ad;%cd' --date=rfc -n 1 " \
+ commit, destdir=repodir, logger=logger)
recordcommit = commit
try:
with transaction.atomic():
for recipe_data in recipes:
_create_upgrade(recipe_data, layerbranch, recordcommit, title,
info, logger, initial=options.initial)
if options.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
tinfoil.shutdown()
if __name__=="__main__":
logger = None
try:
utils.setup_django()
from django.db import transaction
import settings
logger = get_logger("HistoryUpgrade", settings)
parser = optparse.OptionParser(usage = """%prog [options] <layerbranchid> <commit>""")
parser.add_option("-i", "--initial",
help = "Do initial population of upgrade histories (and specify comment)",
action="store", dest="initial", default='')
parser.add_option("--bitbake-rev",
help = "Use the specified bitbake revision instead of the most recent one at the metadata commit date",
action="store", dest="bitbake_rev", default='')
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("--dry-run",
help = "Do not write any data back to the database",
action="store_true", dest="dry_run", default=False)
options, args = parser.parse_args(sys.argv)
logger.setLevel(options.loglevel)
if len(args) < 2:
logger.error('Please specify layerbranch ID')
sys.exit(1)
if len(args) < 3:
logger.error('Please specify commit')
sys.exit(1)
generate_history(options, int(args[1]), args[2], logger)
except KeyboardInterrupt:
if logger:
logger.info('Update interrupted, exiting')
sys.exit(254)