mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-05 05:04:46 +02:00
2314fb2811
Added SPDX identifiers to all .py files except those in migrations directory. Fixes: [YOCTO #13527] Signed-off-by: Meh Mbeh Ida Delphine <idadelm@gmail.com> Signed-off-by: Paul Eggleton <bluelightning@bluelightning.org>
618 lines
28 KiB
Python
618 lines
28 KiB
Python
# Internal script called by rrs_upgrade_history.py
|
|
#
|
|
# To detect package versions of the recipes the script uses the name of the recipe.
|
|
#
|
|
# Copyright (C) 2015, 2018 Intel Corporation
|
|
# Authors: Anibal Limon <anibal.limon@linux.intel.com>
|
|
# Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
#
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
|
|
import sys
|
|
import os
|
|
import optparse
|
|
import logging
|
|
import re
|
|
from distutils.version import LooseVersion
|
|
import git
|
|
from datetime import datetime
|
|
import calendar
|
|
from email.utils import parsedate_tz
|
|
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__))))
|
|
from common import common_setup, get_pv_type, load_recipes, \
|
|
get_logger, DryRunRollbackException
|
|
|
|
common_setup()
|
|
from layerindex import utils, recipeparse
|
|
from layerindex.update_layer import split_recipe_fn
|
|
|
|
|
|
def rfc2822_time_to_utc_datetime(ds):
|
|
tt = parsedate_tz(ds)
|
|
if tt is None:
|
|
return None
|
|
timestamp = calendar.timegm(tt) - tt[9]
|
|
return datetime.utcfromtimestamp(timestamp)
|
|
|
|
|
|
oecore_bad_revs = {
|
|
'40a904bf8bc1279c3da0893c003f740f1d2066c2': [
|
|
'2dfbff215f3567252fdfbd5704e6740a30ad41b4',
|
|
'543e39ad5e2baa0f1ece013a89483783e6b15dd9',
|
|
'a39830b77f567e2361f1ced49bfdce52591e220c',
|
|
'c88304a78e528596ca481cabe273749c286c352a',
|
|
'21d015f6c9927598d64c48c925638619b25cf232',
|
|
'da29440633706fb7a346391d97894d6f2cbb0d01',
|
|
'b9b254da08c1db94ac9ded5f67d7e2e82e3b9be7',
|
|
'12e859dfb70f8aae40edfd88b143b6c771f4e1a6',
|
|
'ec957a03010949a93fbebd3e7b8b924ebc055ef7',
|
|
'2f0dd67a5a8d4269f5155004d532d8fa972b3223',
|
|
'1c2197f96d69547e10b74dc722d9a569d9a2b2b6',
|
|
'7aa94abac09be6beb7ce14a2b9a409e934465706',
|
|
'1a0ee6b0f194807b9eac1207c43ba3fae4d1f94c',
|
|
'bbd9524256461f1bcafd4103edd575e668de76f8',
|
|
'68e0080a924654245f04cf92c2579abd9e5bc658',
|
|
'1ed072515f2a23de75ee56b86d8607c85b42605c',
|
|
'cb3c0343becc8bb2ebf4e9c12782c509a3d7754d',
|
|
'94793d08b0087b7f579b2ca5adae3343864e5f66',
|
|
'802c4029f90cee3027b6bc62c5201e8b29f02557',
|
|
'b992be195821e110691434e4a743b753bc04b3c4',
|
|
'6414d23cd23fc6ed2d31a7b55fce1be82a09ae67',
|
|
'458c835fe05279467ab781aab811498ab80f6904',
|
|
'31c9b42aaeef3ad66e05e51b8209e87f2a22f091',
|
|
'c70b70f045a5ccf62b19060f3438b38d9914e9a2',
|
|
'af4f0d44acef328245dfe1bd102bb5e61293ee2d',
|
|
'747c7dc8702d2241475894876d06a2f1f2b29fed',
|
|
'369bbf393438ae4a76ab0d1817463c6f735816ea',
|
|
'dd5208ae22d47504443785daece4bff6331d8904',
|
|
'c9fdf3d046606a0becb2e6b566a481c483b9021a',
|
|
'13269dfbbc62faef32595343dc78250fdb2a2946',
|
|
'270a1e9bcf26a43f5cbdc5b901c4c6f79495311d',
|
|
'15e876ada73fe8e98284d14dec166007b5767f19',
|
|
'647db1d9eb65b225ffbb6953f796232026bfa935',
|
|
'75529d384bfeaf52befccb892cf41f22dc02668b',
|
|
'ec9fcdf14d3e2aefc5af1e53a69f056db6ea83f5',
|
|
'84980150ff4a2c27acd1f27123f200e03bee8c4b',
|
|
'c93dc7dd18a752d9523e11c6c4dce1908b5970b4',
|
|
'2d9a8a5539342faa1827f4902b1095a9f3448c66',
|
|
'b2cd021887e12d9f5b8ba48d9be3c2f2119c8e2a',
|
|
'da3659155cd1825a4a8d3d7c5288b4273714de15',
|
|
'4af90876914e5f2ccc5b7f833cd43c239c2dac55',
|
|
'8a771f22980f766b71f3ea0825568fc5c669e444',
|
|
'b0338efcdabeec79c568c74b6888d7d523e8e9dc',
|
|
'f3f394913b4e4a7c601ad1158faaf8b9d493e1c7',
|
|
'b3e246fef166030f327b5a852718ea907ada1759',
|
|
'a8a2c5ec891286a1e7fd5ebdd33565f9ae3965c2',
|
|
'ee48cb68e5d91ba108cccdabce003233290ba816',
|
|
'088814ef79015d9df0c8c8bc61219507cfe52ad5',
|
|
'c03cef42e079e4ed3d1e4f401722778157158bd6',
|
|
'bcdaa93dc70411da8876364ae67d0bf2456a3611',
|
|
'e8dfe9799e473e0ba911a0670aa23e8e8d700223',
|
|
'e38e56e28f2090e2b8013546f4dd76da8d59f766',
|
|
'85981cbbf0ce48a6d82bc39248afa9540ca858d8',
|
|
'147f5a665fe5073027d92e4acac631f15f08f79f',
|
|
'b503b1fe9a71f70726c92f46a71fc49615256fce',
|
|
'4972faf1bf20f07a1c1f608bc421c6fd05651594',
|
|
'309a02931779f32d1139cc1169a039cbe4638706',
|
|
],
|
|
'fcd6b38bab8517d83e1ed48eef1bca9a9a190f57': [
|
|
'dad9617809c60ec5f11d4780b0afa1cffa1efed5',
|
|
'0a064f2216895db0181ee033a785328e704ddc0b',
|
|
'de6e98f272e623ce72e724e66920eecf10cb2d41',
|
|
'30d02e2aa2d42fdf76271234b2dc9f37bc46b250',
|
|
'e8cfab060f4ff3c4c16387871354d407910e87aa',
|
|
'25d4d8274bac696a484f83d7f3ada778cf95f4d0',
|
|
'210e290c9251839dc74e3aabdcea3655dd707a50',
|
|
'23c27d9d936efaa17da00525f1d2e2f98c53abc7',
|
|
'ea6245d2383e2ba905ef9f1ba210e5dadc779ad8',
|
|
'ecfcc5dad20943b762a741546732a6c447265251',
|
|
'd022b4335100612d6596cc4c4956cb98ed5873cc',
|
|
'caebd862bac7eed725e0f0321bf50793671b5312',
|
|
'2476bdcbef591e951d11d57d53f1315848758571',
|
|
'bb4685af1bffe17b3aa92a6d21398f38a44ea874',
|
|
'737a095fcde773a36e0fee1f27b74aaa88062386',
|
|
'3dd26cd6b3d731f7698f6fbcd1947969f360cdc4',
|
|
],
|
|
}
|
|
oecore_bad_revs_2 = [
|
|
'2dfbff215f3567252fdfbd5704e6740a30ad41b4',
|
|
'543e39ad5e2baa0f1ece013a89483783e6b15dd9',
|
|
'a39830b77f567e2361f1ced49bfdce52591e220c',
|
|
'c88304a78e528596ca481cabe273749c286c352a',
|
|
'21d015f6c9927598d64c48c925638619b25cf232',
|
|
'da29440633706fb7a346391d97894d6f2cbb0d01',
|
|
'b9b254da08c1db94ac9ded5f67d7e2e82e3b9be7',
|
|
'12e859dfb70f8aae40edfd88b143b6c771f4e1a6',
|
|
'ec957a03010949a93fbebd3e7b8b924ebc055ef7',
|
|
'2f0dd67a5a8d4269f5155004d532d8fa972b3223',
|
|
'1c2197f96d69547e10b74dc722d9a569d9a2b2b6',
|
|
'7aa94abac09be6beb7ce14a2b9a409e934465706',
|
|
'1a0ee6b0f194807b9eac1207c43ba3fae4d1f94c',
|
|
'bbd9524256461f1bcafd4103edd575e668de76f8',
|
|
'68e0080a924654245f04cf92c2579abd9e5bc658',
|
|
'1ed072515f2a23de75ee56b86d8607c85b42605c',
|
|
'cb3c0343becc8bb2ebf4e9c12782c509a3d7754d',
|
|
'94793d08b0087b7f579b2ca5adae3343864e5f66',
|
|
'802c4029f90cee3027b6bc62c5201e8b29f02557',
|
|
'b992be195821e110691434e4a743b753bc04b3c4',
|
|
'6414d23cd23fc6ed2d31a7b55fce1be82a09ae67',
|
|
'458c835fe05279467ab781aab811498ab80f6904',
|
|
'31c9b42aaeef3ad66e05e51b8209e87f2a22f091',
|
|
'c70b70f045a5ccf62b19060f3438b38d9914e9a2',
|
|
'af4f0d44acef328245dfe1bd102bb5e61293ee2d',
|
|
'747c7dc8702d2241475894876d06a2f1f2b29fed',
|
|
'369bbf393438ae4a76ab0d1817463c6f735816ea',
|
|
'dd5208ae22d47504443785daece4bff6331d8904',
|
|
'c9fdf3d046606a0becb2e6b566a481c483b9021a',
|
|
'13269dfbbc62faef32595343dc78250fdb2a2946',
|
|
'270a1e9bcf26a43f5cbdc5b901c4c6f79495311d',
|
|
'15e876ada73fe8e98284d14dec166007b5767f19',
|
|
'647db1d9eb65b225ffbb6953f796232026bfa935',
|
|
'75529d384bfeaf52befccb892cf41f22dc02668b',
|
|
'ec9fcdf14d3e2aefc5af1e53a69f056db6ea83f5',
|
|
'84980150ff4a2c27acd1f27123f200e03bee8c4b',
|
|
'c93dc7dd18a752d9523e11c6c4dce1908b5970b4',
|
|
'2d9a8a5539342faa1827f4902b1095a9f3448c66',
|
|
'b2cd021887e12d9f5b8ba48d9be3c2f2119c8e2a',
|
|
'da3659155cd1825a4a8d3d7c5288b4273714de15',
|
|
'4af90876914e5f2ccc5b7f833cd43c239c2dac55',
|
|
'8a771f22980f766b71f3ea0825568fc5c669e444',
|
|
'b0338efcdabeec79c568c74b6888d7d523e8e9dc',
|
|
'f3f394913b4e4a7c601ad1158faaf8b9d493e1c7',
|
|
'b3e246fef166030f327b5a852718ea907ada1759',
|
|
'a8a2c5ec891286a1e7fd5ebdd33565f9ae3965c2',
|
|
'ee48cb68e5d91ba108cccdabce003233290ba816',
|
|
'088814ef79015d9df0c8c8bc61219507cfe52ad5',
|
|
'c03cef42e079e4ed3d1e4f401722778157158bd6',
|
|
'bcdaa93dc70411da8876364ae67d0bf2456a3611',
|
|
'e8dfe9799e473e0ba911a0670aa23e8e8d700223',
|
|
'e38e56e28f2090e2b8013546f4dd76da8d59f766',
|
|
'85981cbbf0ce48a6d82bc39248afa9540ca858d8',
|
|
'147f5a665fe5073027d92e4acac631f15f08f79f',
|
|
'b503b1fe9a71f70726c92f46a71fc49615256fce',
|
|
'4972faf1bf20f07a1c1f608bc421c6fd05651594',
|
|
'309a02931779f32d1139cc1169a039cbe4638706',
|
|
]
|
|
|
|
|
|
"""
|
|
Store upgrade into RecipeUpgrade model.
|
|
"""
|
|
def _save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, commit, title, info, filepath, logger, upgrade_type=None, orig_filepath=None, prev_version=None):
|
|
from rrs.models import Maintainer, RecipeUpgrade
|
|
|
|
maintainer_name = info.split(';')[0]
|
|
maintainer_email = info.split(';')[1]
|
|
author_date = info.split(';')[2]
|
|
commit_date = info.split(';')[3]
|
|
|
|
maintainer = Maintainer.create_or_update(maintainer_name, maintainer_email)
|
|
|
|
upgrade = RecipeUpgrade()
|
|
upgrade.recipesymbol = recipesymbol
|
|
upgrade.maintainer = maintainer
|
|
upgrade.author_date = rfc2822_time_to_utc_datetime(author_date)
|
|
upgrade.commit_date = rfc2822_time_to_utc_datetime(commit_date)
|
|
upgrade.version = pv
|
|
upgrade.srcrev = srcrev
|
|
upgrade.sha1 = commit
|
|
upgrade.title = title.strip()
|
|
upgrade.filepath = filepath
|
|
if upgrade_type:
|
|
upgrade.upgrade_type = upgrade_type
|
|
if orig_filepath:
|
|
upgrade.orig_filepath = orig_filepath
|
|
if prev_version:
|
|
upgrade.prev_version = prev_version
|
|
upgrade.license = license
|
|
upgrade.regroup()
|
|
upgrade.save()
|
|
|
|
"""
|
|
Create upgrade receives new recipe_data and cmp versions.
|
|
"""
|
|
def _create_upgrade(recipe_data, layerbranch, ct, title, info, filepath, logger, initial=False, orig_filepath=None):
|
|
from rrs.models import RecipeUpgrade, RecipeSymbol, RecipeUpgradeGroupRule
|
|
from bb.utils import vercmp_string
|
|
|
|
pn = recipe_data.getVar('PN', True)
|
|
pv = recipe_data.getVar('PV', True)
|
|
srcrev = recipe_data.getVar('SRCREV', True)
|
|
if srcrev == 'INVALID':
|
|
srcrev = ''
|
|
license = recipe_data.getVar('LICENSE', True)
|
|
|
|
if '..' in pv or pv.endswith('.'):
|
|
logger.warn('Invalid version for recipe %s in commit %s, ignoring' % (recipe_data.getVar('FILE', True), ct))
|
|
return
|
|
|
|
summary = recipe_data.getVar('SUMMARY', True) or recipe_data.getVar('DESCRIPTION', True)
|
|
recipesymbol = RecipeSymbol.symbol(recipe_data.getVar('PN', True), layerbranch, summary=summary)
|
|
|
|
all_rupgrades = RecipeUpgrade.objects.filter(recipesymbol=recipesymbol).exclude(sha1=ct)
|
|
rupgrades = all_rupgrades
|
|
group = RecipeUpgradeGroupRule.group_for_params(recipesymbol, pv, license)
|
|
if group:
|
|
rupgrades = all_rupgrades.filter(group=group)
|
|
latest_upgrade = rupgrades.order_by('-commit_date', '-id').first()
|
|
if latest_upgrade:
|
|
prev_pv = latest_upgrade.version
|
|
prev_srcrev = latest_upgrade.srcrev
|
|
else:
|
|
prev_pv = None
|
|
prev_srcrev = ''
|
|
|
|
if prev_pv is None:
|
|
logger.debug("%s: Initial upgrade ( -> %s)." % (pn, pv))
|
|
_save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger)
|
|
else:
|
|
from common import get_recipe_pv_without_srcpv
|
|
|
|
(ppv, _, _) = get_recipe_pv_without_srcpv(prev_pv,
|
|
get_pv_type(prev_pv))
|
|
(npv, _, _) = get_recipe_pv_without_srcpv(pv,
|
|
get_pv_type(pv))
|
|
|
|
try:
|
|
vercmp_result = 0
|
|
if not (npv == 'git' or ppv == 'git'):
|
|
vercmp_result = vercmp_string(ppv, npv)
|
|
|
|
if npv == 'git':
|
|
logger.debug("%s: Avoiding upgrade to unversioned git." % pn)
|
|
elif ppv == 'git' or vercmp_result != 0 or srcrev != prev_srcrev or latest_upgrade.upgrade_type == 'R':
|
|
if initial is True:
|
|
logger.debug("%s: Update initial upgrade ( -> %s)." % \
|
|
(pn, pv))
|
|
latest_upgrade.filepath = filepath
|
|
latest_upgrade.version = pv
|
|
latest_upgrade.save()
|
|
else:
|
|
# Check if the "new" version is already in the database
|
|
same_pv_upgrade = all_rupgrades.filter(version=pv).order_by('-commit_date', '-id').first()
|
|
if same_pv_upgrade and \
|
|
not all_rupgrades.filter(prev_version=pv, commit_date__gt=same_pv_upgrade.commit_date).exists() \
|
|
and \
|
|
not all_rupgrades.filter(upgrade_type__in=['R', 'N'], commit_date__gt=same_pv_upgrade.commit_date).exists():
|
|
# The "previous" recipe is still present, we won't call this an upgrade
|
|
logger.debug('%s: new version %s already exists' % (pn, pv))
|
|
return
|
|
upgrade_type = 'U'
|
|
if vercmp_result == 1:
|
|
upgrade_type = 'D'
|
|
op = {'U': 'upgrade', 'D': 'downgrade'}[upgrade_type]
|
|
logger.debug("%s: detected %s (%s -> %s)" \
|
|
" in ct %s." % (pn, op, prev_pv, pv, ct))
|
|
_save_upgrade(recipesymbol, layerbranch, pv, srcrev, license, ct, title, info, filepath, logger, upgrade_type=upgrade_type, orig_filepath=orig_filepath, prev_version=prev_pv)
|
|
except KeyboardInterrupt:
|
|
raise
|
|
except Exception as e:
|
|
logger.error("%s: fail to detect upgrade (%s -> %s)" \
|
|
" in ct %s: %s" % (pn, prev_pv, pv, ct, str(e)))
|
|
|
|
|
|
"""
|
|
Returns a list containing the fullpaths to the recipes from a commit.
|
|
"""
|
|
def _get_recipes_filenames(ct, repo, repodir, layersubdir_start, logger):
|
|
import glob
|
|
ct_files = []
|
|
deleted = []
|
|
moved_files = []
|
|
added_files = []
|
|
|
|
incdirs = []
|
|
commitobj = repo.commit(ct)
|
|
for parent in commitobj.parents:
|
|
diff = parent.diff(commitobj)
|
|
for diffitem in diff:
|
|
if layersubdir_start and not (diffitem.a_path.startswith(layersubdir_start) or diffitem.b_path.startswith(layersubdir_start)):
|
|
# Not in this layer, skip it
|
|
continue
|
|
if diffitem.a_path.startswith(layersubdir_start + 'lib/') or diffitem.b_path.startswith(layersubdir_start + 'lib/'):
|
|
# A little bit hacky, but we pick up templates otherwise
|
|
continue
|
|
|
|
(typename, _, _) = recipeparse.detect_file_type(diffitem.a_path,
|
|
layersubdir_start)
|
|
|
|
if not diffitem.b_path or diffitem.deleted_file or not diffitem.b_path.startswith(layersubdir_start):
|
|
# Deleted, or moved out of the layer (which we treat as a delete)
|
|
if typename == 'recipe':
|
|
deleted.append(diffitem.a_path)
|
|
continue
|
|
|
|
if typename == 'recipe':
|
|
(to_typename, _, _) = recipeparse.detect_file_type(diffitem.b_path,
|
|
layersubdir_start)
|
|
if to_typename == 'recipe':
|
|
ct_files.append(os.path.join(repodir, diffitem.b_path))
|
|
if diffitem.a_path is None or diffitem.new_file:
|
|
added_files.append(diffitem.b_path)
|
|
if diffitem.a_path != diffitem.b_path:
|
|
moved_files.append((diffitem.a_path, diffitem.b_path))
|
|
elif typename == 'incfile':
|
|
fpath = os.path.dirname(os.path.join(repodir, diffitem.a_path))
|
|
if not fpath in incdirs:
|
|
incdirs.append(fpath)
|
|
|
|
for fpath in incdirs:
|
|
# Let's just assume that all .bb files next to a .inc need to be checked
|
|
for f in glob.glob(os.path.join(fpath, '*.bb')):
|
|
if not f in ct_files:
|
|
ct_files.append(f)
|
|
|
|
# Check moves for recipe -> inc with an added recipe
|
|
# (i.e. the move should really be to the newly added recipe)
|
|
# example: d5a95dc8985a42bb7e50bc4e7dc6b012d711ff08 in OE-Core
|
|
for i,(a,b) in enumerate(moved_files):
|
|
if b.endswith('.inc'):
|
|
for af in added_files:
|
|
# This is naive, but good enough
|
|
if af.rsplit('_')[0] == a.rsplit('_')[0]:
|
|
moved_files[i] = (a,af)
|
|
|
|
return ct_files, deleted, moved_files
|
|
|
|
|
|
def checkout_layer_deps(layerbranch, commit, fetchdir, logger):
|
|
""" Check out the repositories for a layer and its dependencies """
|
|
|
|
oecore_map = {}
|
|
if layerbranch.layer.name != 'openembedded-core':
|
|
# Filter out some bad commits for OE-Core
|
|
for good_rev, bad_revs in oecore_bad_revs.items():
|
|
for bad_rev in bad_revs:
|
|
oecore_map[bad_rev] = good_rev
|
|
|
|
# Some layers will be in the same repository, so we only want to check those out once
|
|
done_repos = []
|
|
def checkout_layer(lb, lcommit=None, lcommitdate=None, force=False):
|
|
urldir = str(lb.layer.get_fetch_dir())
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
if not repodir in done_repos:
|
|
if not lcommit:
|
|
lcommit = utils.runcmd(['git', 'rev-list', '-1', '--before=%s' % lcommitdate, 'origin/master'], repodir, logger=logger).strip()
|
|
if lb.layer.name == 'openembedded-core':
|
|
lmapcommit = oecore_map.get(lcommit, None)
|
|
if lmapcommit:
|
|
logger.debug('Preferring OE-Core revision %s over %s' % (lmapcommit, lcommit))
|
|
lcommit = lmapcommit
|
|
utils.checkout_repo(repodir, lcommit, logger, force)
|
|
if lcommit in oecore_bad_revs_2:
|
|
# Fix issue that was introduced in 309a02931779f32d1139cc1169a039cbe4638706 and fixed in 40a904bf8bc1279c3da0893c003f740f1d2066c2
|
|
with open(os.path.join(repodir, 'meta/conf/bitbake.conf'), 'r') as f:
|
|
lines = f.readlines()
|
|
lines.insert(0, 'BBINCLUDED ?= ""\n')
|
|
with open(os.path.join(repodir, 'meta/conf/bitbake.conf'), 'w') as f:
|
|
f.writelines(lines)
|
|
done_repos.append(repodir)
|
|
|
|
# We "force" here because it's almost certain we'll be checking out a
|
|
# different revision for the layer itself
|
|
checkout_layer(layerbranch, commit, force=True)
|
|
layer_urldir = str(layerbranch.layer.get_fetch_dir())
|
|
layer_repodir = os.path.join(fetchdir, layer_urldir)
|
|
commitdate = utils.runcmd(['git', 'show', '-s', '--format=%ci'], layer_repodir, logger=logger)
|
|
|
|
for dep in layerbranch.get_recursive_dependencies():
|
|
checkout_layer(dep, lcommitdate=commitdate)
|
|
|
|
return commitdate
|
|
|
|
|
|
def generate_history(options, layerbranch_id, commit, logger):
|
|
from layerindex.models import LayerBranch
|
|
from rrs.models import Release, RecipeUpgrade
|
|
layerbranch = LayerBranch.objects.get(id=layerbranch_id)
|
|
|
|
fetchdir = settings.LAYER_FETCH_DIR
|
|
if not fetchdir:
|
|
logger.error("Please set LAYER_FETCH_DIR in settings.py")
|
|
sys.exit(1)
|
|
|
|
layer = layerbranch.layer
|
|
urldir = str(layer.get_fetch_dir())
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
|
|
|
|
if layerbranch.vcs_subdir:
|
|
layersubdir_start = layerbranch.vcs_subdir
|
|
if not layersubdir_start.endswith('/'):
|
|
layersubdir_start += '/'
|
|
else:
|
|
layersubdir_start = ''
|
|
|
|
repo = git.Repo(repodir)
|
|
if repo.bare:
|
|
logger.error('Repository %s is bare, not supported' % repodir)
|
|
sys.exit(1)
|
|
|
|
commitdate = checkout_layer_deps(layerbranch, commit, fetchdir, logger)
|
|
|
|
if options.initial:
|
|
fns = None
|
|
deleted = []
|
|
moved = []
|
|
else:
|
|
if options.filter_files:
|
|
filepath_start = options.filter_files
|
|
else:
|
|
filepath_start = layersubdir_start
|
|
fns, deleted, moved = _get_recipes_filenames(commit, repo, repodir, filepath_start, logger)
|
|
if not (fns or deleted or moved):
|
|
return
|
|
|
|
# setup bitbake
|
|
bitbakepath = os.path.join(fetchdir, 'bitbake')
|
|
if options.bitbake_rev:
|
|
bitbake_rev = options.bitbake_rev
|
|
if not re.match('^[0-9a-f]{40}$', bitbake_rev):
|
|
# Branch name, need to check out detached
|
|
bitbake_rev = 'origin/%s' % bitbake_rev
|
|
else:
|
|
bitbake_rev = utils.runcmd(['git', 'rev-list', '-1', '--before=%s' % commitdate, 'origin/master'], bitbakepath, logger=logger).strip()
|
|
utils.checkout_repo(bitbakepath, bitbake_rev, logger)
|
|
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
|
|
|
|
(tinfoil, d, recipes, tempdir) = load_recipes(layerbranch, bitbakepath,
|
|
fetchdir, settings, logger, recipe_files=fns,
|
|
nocheckout=True)
|
|
try:
|
|
|
|
if options.initial:
|
|
title = options.initial
|
|
info = 'No maintainer;;' + utils.runcmd(['git', 'log', '--format=%ad;%cd', '--date=rfc', '-n', '1', commit], destdir=repodir, logger=logger)
|
|
recordcommit = ''
|
|
else:
|
|
title = utils.runcmd(['git', 'log', '--format=%s', '-n', '1', commit],
|
|
repodir, logger=logger)
|
|
info = utils.runcmd(['git', 'log', '--format=%an;%ae;%ad;%cd', '--date=rfc', '-n', '1', commit], destdir=repodir, logger=logger)
|
|
recordcommit = commit
|
|
|
|
fn_data = {}
|
|
for recipe_data in recipes:
|
|
fn = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
|
|
fn_data[fn] = recipe_data
|
|
|
|
seen_pns = []
|
|
try:
|
|
with transaction.atomic():
|
|
# Handle recipes where PN has changed
|
|
for a, b in moved:
|
|
logger.debug('Move %s -> %s' % (a,b))
|
|
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
|
|
recipe_data = fn_data.get(b, None)
|
|
if recipe_data:
|
|
pn = recipe_data.getVar('PN', True)
|
|
ru = rus.first()
|
|
if ru and ru.recipesymbol.pn != pn:
|
|
# PN has been changed! We need to mark the old record as deleted
|
|
logger.debug('PN changed (with move): %s -> %s' % (ru.recipesymbol.pn, pn))
|
|
if a not in deleted:
|
|
deleted.append(a)
|
|
else:
|
|
logger.warning('Unable to find parsed data for recipe %s' % b)
|
|
|
|
# Handle recipes that exist at this point in time (which may have upgraded)
|
|
for recipe_data in recipes:
|
|
pn = recipe_data.getVar('PN', True)
|
|
filepath = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
|
|
# Check if PN has changed internally
|
|
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date', '-id')
|
|
deleted_pns = rus.filter(upgrade_type__in=['R', 'N']).values_list('recipesymbol__pn', flat=True).distinct()
|
|
for ru in rus:
|
|
if ru.recipesymbol.pn != pn and ru.recipesymbol.pn not in deleted_pns and ru.upgrade_type not in ['R', 'N']:
|
|
# PN changed (set within recipe), we need to mark the old recipe as deleted
|
|
logger.debug('PN changed (without move): %s -> %s' % (ru.recipesymbol.pn, pn))
|
|
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, ru.filepath, logger, upgrade_type='R')
|
|
orig_filepath = None
|
|
for a, b in moved:
|
|
if b == filepath:
|
|
orig_filepath = a
|
|
break
|
|
_create_upgrade(recipe_data, layerbranch, recordcommit, title,
|
|
info, filepath, logger, initial=options.initial, orig_filepath=orig_filepath)
|
|
seen_pns.append(pn)
|
|
|
|
# Handle recipes that have been moved without it being an upgrade/delete
|
|
for a, b in moved:
|
|
if a not in deleted:
|
|
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
|
|
if rus:
|
|
ru = rus.first()
|
|
if not RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, filepath=b).exists():
|
|
# Need to record the move, otherwise we won't be able to
|
|
# find the record if we need to mark the recipe as deleted later
|
|
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
|
|
|
|
# Handle deleted recipes
|
|
for df in deleted:
|
|
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date', '-id')
|
|
for ru in rus:
|
|
other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gte=ru.commit_date).exclude(filepath=df).order_by('-commit_date', '-id')
|
|
# We make a distinction between deleting just one version and the entire recipe being deleted
|
|
upgrade_type = 'R'
|
|
for other_ru in other_rus:
|
|
if other_ru.upgrade_type == 'R':
|
|
logger.debug('There is a delete: %s' % other_ru)
|
|
upgrade_type = ''
|
|
break
|
|
if os.path.exists(os.path.join(repodir, other_ru.filepath)):
|
|
upgrade_type = 'N'
|
|
if not upgrade_type:
|
|
continue
|
|
if ru.upgrade_type != upgrade_type and ru.recipesymbol.pn not in seen_pns:
|
|
if upgrade_type == 'R':
|
|
finalmsg = ' [FINAL]'
|
|
else:
|
|
finalmsg = ''
|
|
logger.debug("%s: marking as deleted%s (%s)" % (ru.recipesymbol.pn, finalmsg, ru.filepath))
|
|
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, df, logger, upgrade_type=upgrade_type)
|
|
break
|
|
|
|
if options.dry_run:
|
|
raise DryRunRollbackException
|
|
except DryRunRollbackException:
|
|
pass
|
|
|
|
finally:
|
|
if tinfoil and hasattr(tinfoil, 'shutdown') and (LooseVersion(bb.__version__) > LooseVersion("1.27")):
|
|
tinfoil.shutdown()
|
|
utils.rmtree_force(tempdir)
|
|
|
|
|
|
if __name__=="__main__":
|
|
logger = None
|
|
try:
|
|
utils.setup_django()
|
|
from django.db import transaction
|
|
import settings
|
|
|
|
logger = get_logger("HistoryUpgrade", settings)
|
|
|
|
parser = optparse.OptionParser(usage = """%prog [options] <layerbranchid> <commit>""")
|
|
|
|
parser.add_option("-i", "--initial",
|
|
help = "Do initial population of upgrade histories (and specify comment)",
|
|
action="store", dest="initial", default='')
|
|
|
|
parser.add_option("--bitbake-rev",
|
|
help = "Use the specified bitbake revision instead of the most recent one at the metadata commit date",
|
|
action="store", dest="bitbake_rev", default='')
|
|
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
|
|
parser.add_option("--dry-run",
|
|
help = "Do not write any data back to the database",
|
|
action="store_true", dest="dry_run", default=False)
|
|
|
|
parser.add_option("-F", "--filter-files",
|
|
help="Only operate on a specified subset of files (wildcards allowed)",
|
|
action="store", dest="filter_files", default='')
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
if len(args) < 2:
|
|
logger.error('Please specify layerbranch ID')
|
|
sys.exit(1)
|
|
|
|
if len(args) < 3:
|
|
logger.error('Please specify commit')
|
|
sys.exit(1)
|
|
|
|
generate_history(options, int(args[1]), args[2], logger)
|
|
except KeyboardInterrupt:
|
|
if logger:
|
|
logger.info('Update interrupted, exiting')
|
|
sys.exit(254)
|
|
|