bitbake/fetch2: Rewrite and improve exception handling, reusing core functions for common operations where possible

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/lib/bb/fetch2/__init__.py

@@ -39,27 +39,61 @@ logger = logging.getLogger("BitBake.Fetch")
 
 class BBFetchException(Exception):
     """Class all fetch exceptions inherit from"""
+    def __init__(self, message):
+         self.msg = message
+         Exception.__init__(self, message)
+
+    def __str__(self):
+         return self.msg
 
 class MalformedUrl(BBFetchException):
     """Exception raised when encountering an invalid url"""
+    def __init__(self, url):
+         self.msg = "The URL: '%s' is invalid and cannot be interpreted" % url
+         self.url = url
+         Exception.__init__(self, self.msg)
 
 class FetchError(BBFetchException):
-    """Exception raised when a download fails"""
+    """General fetcher exception when something happens incorrectly"""
+    def __init__(self, message, url = None):
+         self.msg = "Fetcher failure for URL: '%s'. %s" % (url, message)
+         self.url = url
+         Exception.__init__(self, self.msg)
 
 class NoMethodError(BBFetchException):
     """Exception raised when there is no method to obtain a supplied url or set of urls"""
+    def __init__(self, url):
+         self.msg = "Could not find a fetcher which supports the URL: '%s'" % url
+         self.url = url
+         Exception.__init__(self, self.msg)
 
 class MissingParameterError(BBFetchException):
     """Exception raised when a fetch method is missing a critical parameter in the url"""
+    def __init__(self, missing, url):
+         self.msg = "URL: '%s' is missing the required parameter '%s'" % (url, missing)
+         self.url = url
+         self.missing = missing
+         Exception.__init__(self, self.msg)
 
 class ParameterError(BBFetchException):
     """Exception raised when a url cannot be proccessed due to invalid parameters."""
+    def __init__(self, message, url):
+         self.msg = "URL: '%s' has invalid parameters. %s" % (url, message)
+         self.url = url
+         Exception.__init__(self, self.msg)
 
 class MD5SumError(BBFetchException):
-    """Exception raised when a MD5SUM of a file does not match the expected one"""
+    """Exception raised when a MD5 checksum of a file does not match for a downloaded file"""
+    def __init__(self, path, wanted, got, url):
+         self.msg = "File: '%s' has md5 sum %s when %s was expected (from URL: '%s')" % (path, got, wanted, url)
+         self.url = url
+         self.path = path
+         self.wanted = wanted
+         self.got = got
+         Exception.__init__(self, self.msg)
 
-class InvalidSRCREV(BBFetchException):
+class SHA256SumError(MD5SumError):
-    """Exception raised when an invalid SRCREV is encountered"""
+    """Exception raised when a SHA256 checksum of a file does not match for a downloaded file"""
 
 def decodeurl(url):
     """Decodes an URL into the tokens (scheme, network location, path,
@@ -108,8 +142,10 @@ def encodeurl(decoded):
 
     (type, host, path, user, pswd, p) = decoded
 
-    if not type or not path:
+    if not path:
-        raise MissingParameterError("Type or path url components missing when encoding %s" % decoded)
+        raise MissingParameterError('path', "encoded from the data %s" % str(decoded))
+    if not type:
+        raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
     url = '%s://' % type
     if user and type != "file":
         url += "%s" % user
@@ -243,16 +279,14 @@ def verify_checksum(u, ud, d):
                     ud.localpath, ud.md5_name, md5data,
                     ud.sha256_name, sha256data)
         if bb.data.getVar("BB_STRICT_CHECKSUM", d, True) == "1":
-            raise FetchError("No checksum specified for %s." % u)
+            raise FetchError("No checksum specified for %s." % u, u)
         return
 
-    if (ud.md5_expected != md5data or ud.sha256_expected != sha256data):
+    if ud.md5_expected != md5data:
-        logger.error('The checksums for "%s" did not match.\n'
+        raise MD5SumError(ud.localpath, ud.md5_expected, md5data, u)
-                     '  MD5: expected "%s", got "%s"\n'
+
-                     '  SHA256: expected "%s", got "%s"\n',
+    if ud.sha256_expected != sha256data:
-                     ud.localpath, ud.md5_expected, md5data,
+        raise SHA256SumError(ud.localpath, ud.sha256_expected, sha256data, u)
-                     ud.sha256_expected, sha256data)
-        raise FetchError("%s checksum mismatch." % u)
 
 def subprocess_setup():
     import signal
@@ -319,7 +353,7 @@ def download(d, urls = None):
                 localpath = try_mirrors (d, u, mirrors)
 
         if not localpath or not os.path.exists(localpath):
-            raise FetchError("Unable to fetch URL %s from any source." % u)
+            raise FetchError("Unable to fetch URL %s from any source." % u, u)
 
         download_update(localpath, ud.localpath)
 
@@ -365,7 +399,7 @@ def checkstatus(d, urls = None):
                 ret = try_mirrors (d, u, mirrors, True)
 
         if not ret:
-            raise FetchError("URL %s doesn't work" % u)
+            raise FetchError("URL %s doesn't work" % u, u)
 
 def localpaths(d):
     """
@@ -403,8 +437,7 @@ def get_srcrev(d):
             scms.append(u)
 
     if len(scms) == 0:
-        logger.error("SRCREV was used yet no valid SCM was found in SRC_URI")
+        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
-        raise ParameterError
 
     if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
         return urldata[scms[0]].method.sortable_revision(scms[0], urldata[scms[0]], d, urldata[scms[0]].names[0])
@@ -414,8 +447,7 @@ def get_srcrev(d):
     #
     format = bb.data.getVar('SRCREV_FORMAT', d, 1)
     if not format:
-        logger.error("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
+        raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.")
-        raise ParameterError
 
     for scm in scms:
         ud = urldata[scm]
@@ -435,11 +467,12 @@ def localpath(url, d):
         return ud[url].localpath
     return url
 
-def runfetchcmd(cmd, d, quiet = False):
+def runfetchcmd(cmd, d, quiet = False, cleanup = []):
     """
     Run cmd returning the command output
     Raise an error if interrupted or cmd fails
     Optionally echo command output to stdout
+    Optionally remove the files/directories listed in cleanup upon failure
     """
 
     # Need to export PATH as binary could be in metadata paths
@@ -474,6 +507,13 @@ def runfetchcmd(cmd, d, quiet = False):
     signal = status >> 8
     exitstatus = status & 0xff
 
+    if (signal or status != 0):
+        for f in cleanup:
+            try:
+                bb.utils.remove(f, True)
+            except OSError:
+                pass
+
         if signal:
             raise FetchError("Fetch command %s failed with signal %s, output:\n%s" % (cmd, signal, output))
         elif status != 0:
@@ -486,10 +526,9 @@ def check_network_access(d, info = ""):
     log remote network access, and error if BB_NO_NETWORK is set
     """
     if bb.data.getVar("BB_NO_NETWORK", d, True) == "1":
-        bb.error("BB_NO_NETWORK is set, but the fetcher code attempted network access with the command %s" % info)
+        raise FetchError("BB_NO_NETWORK is set, but the fetcher code attempted network access with the command %s" % info)
-        raise FetchError("BB_NO_NETWORK violation")
     else:
-        bb.note("Fetcher accessed the network with the command %s" % info)
+        logger.debug(1, "Fetcher accessed the network with the command %s" % info)
 
 def try_mirrors(d, uri, mirrors, check = False, force = False):
     """
@@ -573,7 +612,7 @@ class FetchData(object):
                 break                
 
         if not self.method:
-            raise NoMethodError("Missing implementation for url %s" % url)
+            raise NoMethodError(url)
 
         if self.method.supports_srcrev():
             self.revisions = {}
@@ -658,7 +697,7 @@ class Fetch(object):
         Fetch urls
         Assumes localpath was called first
         """
-        raise NoMethodError("Missing implementation for url")
+        raise NoMethodError(url)
 
     def unpack(self, urldata, rootdir, data):
         import subprocess
@@ -795,7 +834,7 @@ class Fetch(object):
         if not rev:
             rev = data.getVar("SRCREV", d, 1)
         if rev == "INVALID":
-            raise InvalidSRCREV("Please set SRCREV to a valid value")
+            raise FetchError("Please set SRCREV to a valid value", ud.url)
         if rev == "AUTOINC":
             rev = ud.method.latest_revision(ud.url, ud, d, name)
 
@@ -828,14 +867,15 @@ class Fetch(object):
         if not wanted_sum:
             return True
 
-        return wanted_sum == got_sum
+        if wanted_sum != got_sum:
+            raise MD5SumError(ud.localpath, wanted_sum, got_sum, ud.url)
+
     verify_md5sum = staticmethod(verify_md5sum)
 
     def write_md5sum(url, ud, d):
         md5data = bb.utils.md5_file(ud.localpath)
-        # verify the md5sum
+
-        if not Fetch.verify_md5sum(ud, md5data):
+        Fetch.verify_md5sum(ud, md5data)
-            raise MD5SumError(url)
 
         md5out = file(ud.md5, 'w')
         md5out.write(md5data)
@@ -847,7 +887,7 @@ class Fetch(object):
         Look in the cache for the latest revision, if not present ask the SCM.
         """
         if not hasattr(self, "_latest_revision"):
-            raise ParameterError
+            raise ParameterError("The fetcher for this URL does not support _latest_revision", url)
 
         pd = persist_data.persist(d)
         revs = pd['BB_URI_HEADREVS']

bitbake/lib/bb/fetch2/bzr.py

@@ -72,7 +72,7 @@ class Bzr(Fetch):
             elif command is "update":
                 bzrcmd = "%s pull %s --overwrite" % (basecmd, " ".join(options))
             else:
-                raise FetchError("Invalid bzr command %s" % command)
+                raise FetchError("Invalid bzr command %s" % command, ud.url)
 
         return bzrcmd
 
@@ -104,15 +104,7 @@ class Bzr(Fetch):
             tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
 
         # tar them up to a defined filename
-        try:
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath])
-            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
 
     def supports_srcrev(self):
         return True

bitbake/lib/bb/fetch2/cvs.py

@@ -44,7 +44,7 @@ class Cvs(Fetch):
 
     def urldata_init(self, ud, d):
         if not "module" in ud.parm:
-            raise MissingParameterError("cvs method needs a 'module' parameter")
+            raise MissingParameterError("module", ud.url)
         ud.module = ud.parm["module"]
 
         ud.tag = ud.parm.get('tag', "")
@@ -132,7 +132,7 @@ class Cvs(Fetch):
             bb.fetch2.check_network_access(d, cvsupdatecmd)
             # update sources there
             os.chdir(moddir)
-            myret = os.system(cvsupdatecmd)
+            cmd = cvsupdatecmd
         else:
             logger.info("Fetch " + loc)
             # check out sources there
@@ -140,14 +140,12 @@ class Cvs(Fetch):
             os.chdir(pkgdir)
             logger.debug(1, "Running %s", cvscmd)
             bb.fetch2.check_network_access(d, cvscmd)
-            myret = os.system(cvscmd)
+            cmd = cvscmd
 
-        if myret != 0 or not os.access(moddir, os.R_OK):
+        runfetchcmd(cmd, d, cleanup = [moddir])
-            try:
+
-                os.rmdir(moddir)
+        if not os.access(moddir, os.R_OK):
-            except OSError:
+            raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url)
-                pass
-            raise FetchError(ud.module)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata == "keep":
@@ -158,15 +156,11 @@ class Cvs(Fetch):
         # tar them up to a defined filename
         if 'fullpath' in ud.parm:
             os.chdir(pkgdir)
-            myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir))
+            cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)
         else:
             os.chdir(moddir)
             os.chdir('..')
-            myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir)))
+            cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir))
+
+        runfetchcmd(cmd, d, cleanup = [ud.localpath])
 
-        if myret != 0:
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise FetchError(ud.module)

bitbake/lib/bb/fetch2/git.py

@@ -59,7 +59,7 @@ class Git(Fetch):
 
         branches = ud.parm.get("branch", "master").split(',')
         if len(branches) != len(ud.names):
-            raise bb.fetch2.ParameterError("SRC_URI (%) name and branch number mismatch" % ud.url)
+            raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
         ud.branches = {}
         for name in ud.names:
             branch = branches[ud.names.index(name)]
@@ -194,7 +194,7 @@ class Git(Fetch):
         cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branches[name])
         output = runfetchcmd(cmd, d, True)
         if not output:
-            raise bb.fetch2.FetchError("Fetch command %s gave empty output\n" % (cmd))
+            raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, url)
         return output.split()[0]
 
     def _build_revision(self, url, ud, d, name):

bitbake/lib/bb/fetch2/hg.py

@@ -48,7 +48,7 @@ class Hg(Fetch):
         init hg specific variable within url data
         """
         if not "module" in ud.parm:
-            raise MissingParameterError("hg method needs a 'module' parameter")
+            raise MissingParameterError('module', ud.url)
 
         ud.module = ud.parm["module"]
 
@@ -105,7 +105,7 @@ class Hg(Fetch):
         elif command is "update":
             cmd = "%s update -C %s" % (basecmd, " ".join(options))
         else:
-            raise FetchError("Invalid hg command %s" % command)
+            raise FetchError("Invalid hg command %s" % command, ud.url)
 
         return cmd
 
@@ -147,15 +147,7 @@ class Hg(Fetch):
             tar_flags = "--exclude '.hg' --exclude '.hgrags'"
 
         os.chdir(ud.pkgdir)
-        try:
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath])
-            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
 
     def supports_srcrev(self):
         return True

bitbake/lib/bb/fetch2/osc.py

@@ -28,7 +28,7 @@ class Osc(Fetch):
 
     def urldata_init(self, ud, d):
         if not "module" in ud.parm:
-            raise MissingParameterError("osc method needs a 'module' parameter.")
+            raise MissingParameterError('module', ud.url)
 
         ud.module = ud.parm["module"]
 
@@ -73,7 +73,7 @@ class Osc(Fetch):
         elif command is "update":
             osccmd = "%s %s up %s" % (basecmd, config, " ".join(options))
         else:
-            raise FetchError("Invalid osc command %s" % command)
+            raise FetchError("Invalid osc command %s" % command, ud.url)
 
         return osccmd
 
@@ -104,15 +104,7 @@ class Osc(Fetch):
 
         os.chdir(os.path.join(ud.pkgdir + ud.path))
         # tar them up to a defined filename
-        try:
+        runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath])
-            runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
 
     def supports_srcrev(self):
         return False

bitbake/lib/bb/fetch2/perforce.py

@@ -156,8 +156,7 @@ class Perforce(Fetch):
         tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
         tmpfile = tmppipe.readline().strip()
         if not tmpfile:
-            logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+            raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc)
-            raise FetchError(module)
 
         if "label" in parm:
             depot = "%s@%s" % (depot, parm["label"])
@@ -171,8 +170,7 @@ class Perforce(Fetch):
         p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot))
 
         if not p4file:
-            logger.error("Fetch: unable to get the P4 files from %s", depot)
+            raise FetchError("Fetch: unable to get the P4 files from %s" % depot, loc)
-            raise FetchError(module)
 
         count = 0
 
@@ -189,15 +187,9 @@ class Perforce(Fetch):
             count = count + 1
 
         if count == 0:
-            logger.error("Fetch:  No files gathered from the P4 fetch")
+            logger.error()
-            raise FetchError(module)
+            raise FetchError("Fetch: No files gathered from the P4 fetch", loc)
 
-        myret = os.system("tar -czf %s %s" % (ud.localpath, module))
+        runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath])
-        if myret != 0:
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise FetchError(module)
         # cleanup
         bb.utils.prunedir(tmpfile)

bitbake/lib/bb/fetch2/ssh.py

@@ -114,7 +114,5 @@ class SSH(Fetch):
 
         bb.fetch2.check_network_access(d, cmd)
 
-        (exitstatus, output) = commands.getstatusoutput(cmd)
+        runfetchcmd(cmd, d)
-        if exitstatus != 0:
+
-            print(output)
-            raise FetchError('Unable to fetch %s' % url)

bitbake/lib/bb/fetch2/svk.py

@@ -45,7 +45,7 @@ class Svk(Fetch):
     def urldata_init(self, ud, d):
 
         if not "module" in ud.parm:
-            raise MissingParameterError("svk method needs a 'module' parameter")
+            raise MissingParameterError('module', ud.url)
         else:
             ud.module = ud.parm["module"]
 
@@ -75,29 +75,18 @@ class Svk(Fetch):
         tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
         tmpfile = tmppipe.readline().strip()
         if not tmpfile:
-            logger.error("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
+            logger.error()
-            raise FetchError(ud.module)
+            raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc)
 
         # check out sources there
         os.chdir(tmpfile)
         logger.info("Fetch " + loc)
         logger.debug(1, "Running %s", svkcmd)
-        myret = os.system(svkcmd)
+        runfetchcmd(svkcmd, d, cleanup = [tmpfile])
-        if myret != 0:
-            try:
-                os.rmdir(tmpfile)
-            except OSError:
-                pass
-            raise FetchError(ud.module)
 
         os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
         # tar them up to a defined filename
-        myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
+        runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d, cleanup = [ud.localpath])
-        if myret != 0:
+
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise FetchError(ud.module)
         # cleanup
         bb.utils.prunedir(tmpfile)

bitbake/lib/bb/fetch2/svn.py

@@ -47,7 +47,7 @@ class Svn(Fetch):
         init svn specific variable within url data
         """
         if not "module" in ud.parm:
-            raise MissingParameterError("svn method needs a 'module' parameter")
+            raise MissingParameterError('module', ud.url)
 
         ud.module = ud.parm["module"]
 
@@ -118,7 +118,7 @@ class Svn(Fetch):
             elif command is "update":
                 svncmd = "%s update %s" % (basecmd, " ".join(options))
             else:
-                raise FetchError("Invalid svn command %s" % command)
+                raise FetchError("Invalid svn command %s" % command, ud.url)
 
         if svn_rsh:
             svncmd = "svn_RSH=\"%s\" %s" % (svn_rsh, svncmd)
@@ -156,15 +156,7 @@ class Svn(Fetch):
 
         os.chdir(ud.pkgdir)
         # tar them up to a defined filename
-        try:
+        runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath])
-            runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
-        except:
-            t, v, tb = sys.exc_info()
-            try:
-                os.unlink(ud.localpath)
-            except OSError:
-                pass
-            raise t, v, tb
 
     def supports_srcrev(self):
         return True

bitbake/lib/bb/fetch2/wget.py

@@ -72,20 +72,15 @@ class Wget(Fetch):
             # Sanity check since wget can pretend it succeed when it didn't
             # Also, this used to happen if sourceforge sent us to the mirror page
             if not os.path.exists(ud.localpath) and not checkonly:
-                logger.debug(2, "The fetch command for %s returned success but %s doesn't exist?...", uri, ud.localpath)
+                raise FetchError("The fetch command returned success but %s doesn't exist?!" % (uri, ud.localpath), uri)
-                return False
-
-            return True
 
         localdata = data.createCopy(d)
         data.setVar('OVERRIDES', "wget:" + data.getVar('OVERRIDES', localdata), localdata)
         data.update_data(localdata)
 
-        if fetch_uri(uri, ud, localdata):
+        fetch_uri(uri, ud, localdata)
+        
         return True
 
-        raise FetchError(uri)
-
-
     def checkstatus(self, uri, ud, d):
         return self.download(uri, ud, d, True)