mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
2030e815bb
If the system had previously fetched a source repository for use by gitsm, and then the SRCREV was updated and the new commit already existed, the system would not re-evaluate the submodules and update them accordingly. The cause of this issue was that need_update was being used, unmodified, from the base git fetcher. It did not have any knowledge, nor did it care if we were moving commits and needed to re-evaluate what was happening due to this switch. To fix the issue, during the download process we add all processed (by gitsm) srcrevs to the git config file, as bitbake.srcrev. This allows us to use a new need_update function that not only checks if the git commit is present, but if we have previously processed this commit to ensure all of the submodule components are also present. This approach is used, instead of iterating over the submodules in need_update to avoid a potential race condition that has affected us in the past. The need_update is called only with the parent locking. Any time we need to dive into the submodules, we need to lock, and unlock them, at each stage. This opens the possibility of errors in either the code, or unintended race conditions with rm_work. This issue was discovered by William A. Kennington III <wak@google.com>. The included test case was also written by him, and included unmodified. (Bitbake rev: 30fe86d22c239afa75168cc5eb262b880886ef8a) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
226 lines
9.0 KiB
Python
226 lines
9.0 KiB
Python
# ex:ts=4:sw=4:sts=4:et
|
|
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
|
|
"""
|
|
BitBake 'Fetch' git submodules implementation
|
|
|
|
Inherits from and extends the Git fetcher to retrieve submodules of a git repository
|
|
after cloning.
|
|
|
|
SRC_URI = "gitsm://<see Git fetcher for syntax>"
|
|
|
|
See the Git fetcher, git://, for usage documentation.
|
|
|
|
NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your recipe.
|
|
|
|
"""
|
|
|
|
# Copyright (C) 2013 Richard Purdie
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import os
|
|
import bb
|
|
import copy
|
|
from bb.fetch2.git import Git
|
|
from bb.fetch2 import runfetchcmd
|
|
from bb.fetch2 import logger
|
|
from bb.fetch2 import Fetch
|
|
from bb.fetch2 import BBFetchException
|
|
|
|
class GitSM(Git):
|
|
def supports(self, ud, d):
|
|
"""
|
|
Check to see if a given url can be fetched with git.
|
|
"""
|
|
return ud.type in ['gitsm']
|
|
|
|
def process_submodules(self, ud, workdir, function, d):
|
|
"""
|
|
Iterate over all of the submodules in this repository and execute
|
|
the 'function' for each of them.
|
|
"""
|
|
|
|
submodules = []
|
|
paths = {}
|
|
revision = {}
|
|
uris = {}
|
|
subrevision = {}
|
|
|
|
def parse_gitmodules(gitmodules):
|
|
modules = {}
|
|
module = ""
|
|
for line in gitmodules.splitlines():
|
|
if line.startswith('[submodule'):
|
|
module = line.split('"')[1]
|
|
modules[module] = {}
|
|
elif module and line.strip().startswith('path'):
|
|
path = line.split('=')[1].strip()
|
|
modules[module]['path'] = path
|
|
elif module and line.strip().startswith('url'):
|
|
url = line.split('=')[1].strip()
|
|
modules[module]['url'] = url
|
|
return modules
|
|
|
|
# Collect the defined submodules, and their attributes
|
|
for name in ud.names:
|
|
try:
|
|
gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
|
|
except:
|
|
# No submodules to update
|
|
continue
|
|
|
|
for m, md in parse_gitmodules(gitmodules).items():
|
|
try:
|
|
module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
|
|
except:
|
|
# If the command fails, we don't have a valid file to check. If it doesn't
|
|
# fail -- it still might be a failure, see next check...
|
|
module_hash = ""
|
|
|
|
if not module_hash:
|
|
logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
|
|
continue
|
|
|
|
submodules.append(m)
|
|
paths[m] = md['path']
|
|
revision[m] = ud.revisions[name]
|
|
uris[m] = md['url']
|
|
subrevision[m] = module_hash.split()[2]
|
|
|
|
# Convert relative to absolute uri based on parent uri
|
|
if uris[m].startswith('..'):
|
|
newud = copy.copy(ud)
|
|
newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
|
|
uris[m] = Git._get_repo_url(self, newud)
|
|
|
|
for module in submodules:
|
|
# Translate the module url into a SRC_URI
|
|
|
|
if "://" in uris[module]:
|
|
# Properly formated URL already
|
|
proto = uris[module].split(':', 1)[0]
|
|
url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
|
|
else:
|
|
if ":" in uris[module]:
|
|
# Most likely an SSH style reference
|
|
proto = "ssh"
|
|
if ":/" in uris[module]:
|
|
# Absolute reference, easy to convert..
|
|
url = "gitsm://" + uris[module].replace(':/', '/', 1)
|
|
else:
|
|
# Relative reference, no way to know if this is right!
|
|
logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
|
|
url = "gitsm://" + uris[module].replace(':', '/', 1)
|
|
else:
|
|
# This has to be a file reference
|
|
proto = "file"
|
|
url = "gitsm://" + uris[module]
|
|
|
|
url += ';protocol=%s' % proto
|
|
url += ";name=%s" % module
|
|
url += ";subpath=%s" % paths[module]
|
|
|
|
ld = d.createCopy()
|
|
# Not necessary to set SRC_URI, since we're passing the URI to
|
|
# Fetch.
|
|
#ld.setVar('SRC_URI', url)
|
|
ld.setVar('SRCREV_%s' % module, subrevision[module])
|
|
|
|
# Workaround for issues with SRCPV/SRCREV_FORMAT errors
|
|
# error refer to 'multiple' repositories. Only the repository
|
|
# in the original SRC_URI actually matters...
|
|
ld.setVar('SRCPV', d.getVar('SRCPV'))
|
|
ld.setVar('SRCREV_FORMAT', module)
|
|
|
|
function(ud, url, module, paths[module], ld)
|
|
|
|
return submodules != []
|
|
|
|
def need_update(self, ud, d):
|
|
if Git.need_update(self, ud, d):
|
|
return True
|
|
|
|
try:
|
|
# Check for the nugget dropped by the download operation
|
|
known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
|
|
(ud.basecmd), d, workdir=ud.clonedir)
|
|
|
|
if ud.revisions[ud.names[0]] not in known_srcrevs.split():
|
|
return True
|
|
except bb.fetch2.FetchError:
|
|
# No srcrev nuggets, so this is new and needs to be updated
|
|
return True
|
|
|
|
return False
|
|
|
|
def download(self, ud, d):
|
|
def download_submodule(ud, url, module, modpath, d):
|
|
url += ";bareclone=1;nobranch=1"
|
|
|
|
# Is the following still needed?
|
|
#url += ";nocheckout=1"
|
|
|
|
try:
|
|
newfetch = Fetch([url], d, cache=False)
|
|
newfetch.download()
|
|
# Drop a nugget to add each of the srcrevs we've fetched (used by need_update)
|
|
runfetchcmd("%s config --add bitbake.srcrev %s" % \
|
|
(ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
|
|
except Exception as e:
|
|
logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
|
|
raise
|
|
|
|
Git.download(self, ud, d)
|
|
self.process_submodules(ud, ud.clonedir, download_submodule, d)
|
|
|
|
def unpack(self, ud, destdir, d):
|
|
def unpack_submodules(ud, url, module, modpath, d):
|
|
url += ";bareclone=1;nobranch=1"
|
|
|
|
# Figure out where we clone over the bare submodules...
|
|
if ud.bareclone:
|
|
repo_conf = ud.destdir
|
|
else:
|
|
repo_conf = os.path.join(ud.destdir, '.git')
|
|
|
|
try:
|
|
newfetch = Fetch([url], d, cache=False)
|
|
newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', modpath)))
|
|
except Exception as e:
|
|
logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
|
|
raise
|
|
|
|
local_path = newfetch.localpath(url)
|
|
|
|
# Correct the submodule references to the local download version...
|
|
runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
|
|
|
|
if ud.shallow:
|
|
runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
|
|
|
|
# Ensure the submodule repository is NOT set to bare, since we're checking it out...
|
|
try:
|
|
runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
|
|
except:
|
|
logger.error("Unable to set git config core.bare to false for %s" % os.path.join(repo_conf, 'modules', modpath))
|
|
raise
|
|
|
|
Git.unpack(self, ud, destdir, d)
|
|
|
|
ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
|
|
|
|
if not ud.bareclone and ret:
|
|
# Run submodule update, this sets up the directories -- without touching the config
|
|
runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
|