MIRRORS/poky
1
0
Fork 0
mirror of git://git.yoctoproject.org/poky.git synced 2025-07-19 21:09:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

bitbake: fetch: support multiple mirror tarball filenames

Browse Source 
Remove ud.mirrortarball in favor of ud.mirrortarballs. Each tarball will be
attempted, in order, and the first available will be used. This is needed for
git shallow mirror tarball support, as we want to be able to use either
a shallow or full mirror tarball.

(Bitbake rev: 02eebee6709e57b523862257f75929e64f16d6b0)

Signed-off-by: Christopher Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Christopher Larson 2017-05-13 02:46:26 +05:00 committed by Richard Purdie
parent 60ade6074e
commit ab4e578b86
4 changed files with 47 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
bitbake/lib/bb/fetch2/__init__.py
View File

@ -425,7 +425,7 @@ def encodeurl(decoded):
return url return url
def uri_replace(ud, uri_find, uri_replace, replacements, d): def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
if not ud.url or not uri_find or not uri_replace: if not ud.url or not uri_find or not uri_replace:
logger.error("uri_replace: passed an undefined value, not replacing") logger.error("uri_replace: passed an undefined value, not replacing")
return None return None
@ -464,9 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d):
if loc == 2: if loc == 2:
# Handle path manipulations # Handle path manipulations
basename = None basename = None
if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball: if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball:
# If the source and destination url types differ, must be a mirrortarball mapping # If the source and destination url types differ, must be a mirrortarball mapping
basename = os.path.basename(ud.mirrortarball) basename = os.path.basename(mirrortarball)
# Kill parameters, they make no sense for mirror tarballs # Kill parameters, they make no sense for mirror tarballs
uri_decoded[5] = {} uri_decoded[5] = {}
elif ud.localpath and ud.method.supports_checksum(ud): elif ud.localpath and ud.method.supports_checksum(ud):
@ -892,45 +892,47 @@ def build_mirroruris(origud, mirrors, ld):
replacements["BASENAME"] = origud.path.split("/")[-1] replacements["BASENAME"] = origud.path.split("/")[-1]
replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
def adduri(ud, uris, uds, mirrors): def adduri(ud, uris, uds, mirrors, tarballs):
for line in mirrors: for line in mirrors:
try: try:
(find, replace) = line (find, replace) = line
except ValueError: except ValueError:
continue continue
newuri = uri_replace(ud, find, replace, replacements, ld)
if not newuri or newuri in uris or newuri == origud.url:
continue
if not trusted_network(ld, newuri): for tarball in tarballs:
logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) newuri = uri_replace(ud, find, replace, replacements, ld, tarball)
continue if not newuri or newuri in uris or newuri == origud.url:
continue
# Create a local copy of the mirrors minus the current line if not trusted_network(ld, newuri):
# this will prevent us from recursively processing the same line logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri))
# as well as indirect recursion A -> B -> C -> A continue
localmirrors = list(mirrors)
localmirrors.remove(line) # Create a local copy of the mirrors minus the current line
# this will prevent us from recursively processing the same line
# as well as indirect recursion A -> B -> C -> A
localmirrors = list(mirrors)
localmirrors.remove(line)
try:
newud = FetchData(newuri, ld)
newud.setup_localpath(ld)
except bb.fetch2.BBFetchException as e:
logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
logger.debug(1, str(e))
try: try:
# setup_localpath of file:// urls may fail, we should still see newud = FetchData(newuri, ld)
# if mirrors of the url exist newud.setup_localpath(ld)
adduri(newud, uris, uds, localmirrors) except bb.fetch2.BBFetchException as e:
except UnboundLocalError: logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
pass logger.debug(1, str(e))
continue try:
uris.append(newuri) # setup_localpath of file:// urls may fail, we should still see
uds.append(newud) # if mirrors of the url exist
adduri(newud, uris, uds, localmirrors, tarballs)
except UnboundLocalError:
pass
continue
uris.append(newuri)
uds.append(newud)
adduri(newud, uris, uds, localmirrors) adduri(newud, uris, uds, localmirrors, tarballs)
adduri(origud, uris, uds, mirrors) adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None])
return uris, uds return uris, uds
@ -975,8 +977,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
# We may be obtaining a mirror tarball which needs further processing by the real fetcher # We may be obtaining a mirror tarball which needs further processing by the real fetcher
# If that tarball is a local file:// we need to provide a symlink to it # If that tarball is a local file:// we need to provide a symlink to it
dldir = ld.getVar("DL_DIR") dldir = ld.getVar("DL_DIR")
if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \
and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
# Create donestamp in old format to avoid triggering a re-download # Create donestamp in old format to avoid triggering a re-download
if ud.donestamp: if ud.donestamp:
bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
@ -993,7 +995,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
pass pass
if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
origud.method.download(origud, ld) origud.method.download(origud, ld)
if hasattr(origud.method,"build_mirror_data"): if hasattr(origud.method, "build_mirror_data"):
origud.method.build_mirror_data(origud, ld) origud.method.build_mirror_data(origud, ld)
return origud.localpath return origud.localpath
# Otherwise the result is a local file:// and we symlink to it # Otherwise the result is a local file:// and we symlink to it
@ -1190,7 +1192,7 @@ class FetchData(object):
self.localfile = "" self.localfile = ""
self.localpath = None self.localpath = None
self.lockfile = None self.lockfile = None
self.mirrortarball = None self.mirrortarballs = []
self.basename = None self.basename = None
self.basepath = None self.basepath = None
(self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))

5
bitbake/lib/bb/fetch2/git.py
View File

@ -205,8 +205,9 @@ class Git(FetchMethod):
if ud.rebaseable: if ud.rebaseable:
for name in ud.names: for name in ud.names:
gitsrcname = gitsrcname + '_' + ud.revisions[name] gitsrcname = gitsrcname + '_' + ud.revisions[name]
ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname mirrortarball = 'git2_%s.tar.gz' % gitsrcname
ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
ud.mirrortarballs = [mirrortarball]
gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/") gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
ud.clonedir = os.path.join(gitdir, gitsrcname) ud.clonedir = os.path.join(gitdir, gitsrcname)

5
bitbake/lib/bb/fetch2/hg.py
View File

@ -76,8 +76,9 @@ class Hg(FetchMethod):
# Create paths to mercurial checkouts # Create paths to mercurial checkouts
hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
ud.host, ud.path.replace('/', '.')) ud.host, ud.path.replace('/', '.'))
ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname mirrortarball = 'hg_%s.tar.gz' % hgsrcname
ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
ud.mirrortarballs = [mirrortarball]
hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/") hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
ud.pkgdir = os.path.join(hgdir, hgsrcname) ud.pkgdir = os.path.join(hgdir, hgsrcname)

7
bitbake/lib/bb/fetch2/npm.py
View File

@ -91,9 +91,10 @@ class Npm(FetchMethod):
ud.prefixdir = prefixdir ud.prefixdir = prefixdir
ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
ud.mirrortarball = ud.mirrortarball.replace('/', '-') mirrortarball = ud.mirrortarball.replace('/', '-')
ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
ud.mirrortarballs = [mirrortarball]
def need_update(self, ud, d): def need_update(self, ud, d):
if os.path.exists(ud.localpath): if os.path.exists(ud.localpath):