bitbake-dev: Sync with upstream

Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>

bitbake-dev/ChangeLog

@@ -169,6 +169,13 @@ Changes in Bitbake 1.9.x:
 	  proxies to work better. (from Poky)
 	- Also allow user and pswd options in SRC_URIs globally (from Poky)
 	- Improve proxy handling when using mirrors (from Poky)
+	- Add bb.utils.prune_suffix function
+	- Fix hg checkouts of specific revisions (from Poky)
+	- Fix wget fetching of urls with parameters specified (from Poky)
+	- Add username handling to git fetcher (from Poky)
+	- Set HOME environmental variable when running fetcher commands (from Poky)
+	- Make sure allowed variables inherited from the environment are exported again (from Poky)
+	- When running a stage task in bbshell, run populate_staging, not the stage task (from Poky)
 
 Changes in Bitbake 1.8.0:
 	- Release 1.7.x as a stable series

bitbake-dev/lib/bb/COW.py

@@ -23,10 +23,8 @@
 # Assign a file to __warn__ to get warnings about slow operations.
 #
 
-from inspect import getmro
-
 import copy
-import types, sets
+import types
 types.ImmutableTypes = tuple([ \
     types.BooleanType, \
     types.ComplexType, \
@@ -35,7 +33,7 @@ types.ImmutableTypes = tuple([ \
     types.LongType, \
     types.NoneType, \
     types.TupleType, \
-    sets.ImmutableSet] + \
+    frozenset] + \
     list(types.StringTypes))
 
 MUTABLE = "__mutable__"

bitbake-dev/lib/bb/__init__.py

@@ -1130,4 +1130,5 @@ def dep_opconvert(mysplit, myuse):
 
 if __name__ == "__main__":
     import doctest, bb
+    bb.msg.set_debug_level(0)
     doctest.testmod(bb)

bitbake-dev/lib/bb/cache.py

@@ -31,7 +31,6 @@
 import os, re
 import bb.data
 import bb.utils
-from sets import Set
 
 try:
     import cPickle as pickle
@@ -525,6 +524,6 @@ class CacheData:
         (set elsewhere)
         """
         self.ignored_dependencies = []
-        self.world_target = Set()
+        self.world_target = set()
         self.bbfile_priority = {}
         self.bbfile_config_priorities = []

bitbake-dev/lib/bb/cooker.py

@@ -26,7 +26,6 @@ import sys, os, getopt, glob, copy, os.path, re, time
 import bb
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
 from bb import xmlrpcserver, command
-from sets import Set
 import itertools, sre_constants
 
 class MultipleMatches(Exception):
@@ -97,7 +96,7 @@ class BBCooker:
             self.configuration.cmd = bb.data.getVar("BB_DEFAULT_TASK", self.configuration.data, True) or "build"
 
         bbpkgs = bb.data.getVar('BBPKGS', self.configuration.data, True)
-        if bbpkgs:
+        if bbpkgs and len(self.configuration.pkgs_to_build) == 0:
             self.configuration.pkgs_to_build.extend(bbpkgs.split())
 
         #
@@ -635,7 +634,7 @@ class BBCooker:
 
         # Tweak some variables
         item = self.bb_cache.getVar('PN', fn, True)
-        self.status.ignored_dependencies = Set()
+        self.status.ignored_dependencies = set()
         self.status.bbfile_priority[fn] = 1
 
         # Remove external dependencies
@@ -762,7 +761,7 @@ class BBCooker:
         self.status = bb.cache.CacheData()
 
         ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
-        self.status.ignored_dependencies = Set(ignore.split())
+        self.status.ignored_dependencies = set(ignore.split())
 
         for dep in self.configuration.extra_assume_provided:
             self.status.ignored_dependencies.add(dep)
@@ -836,7 +835,11 @@ class BBCooker:
                 if dirfiles:
                     newfiles += dirfiles
                     continue
-            newfiles += glob.glob(f) or [ f ]
+            else:
+                globbed = glob.glob(f)
+                if not globbed and os.path.exists(f):
+                    globbed = [f]
+                newfiles += globbed
 
         bbmask = bb.data.getVar('BBMASK', self.configuration.data, 1)
 
@@ -849,9 +852,8 @@ class BBCooker:
             bb.msg.fatal(bb.msg.domain.Collection, "BBMASK is not a valid regular expression.")
 
         finalfiles = []
-        for i in xrange( len( newfiles ) ):
+        for f in newfiles:
-            f = newfiles[i]
+            if bbmask_compiled.search(f):
-            if bbmask and bbmask_compiled.search(f):
                 bb.msg.debug(1, bb.msg.domain.Collection, "skipping masked file %s" % f)
                 masked += 1
                 continue

bitbake-dev/lib/bb/data.py

@@ -37,7 +37,7 @@ the speed is more critical here.
 #
 #Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import sys, os, re, time, types
+import sys, os, re, types
 if sys.argv[0][-5:] == "pydoc":
     path = os.path.dirname(os.path.dirname(sys.argv[1]))
 else:
@@ -553,7 +553,9 @@ def inherits_class(klass, d):
 def _test():
     """Start a doctest run on this module"""
     import doctest
+    import bb
     from bb import data
+    bb.msg.set_debug_level(0)
     doctest.testmod(data)
 
 if __name__ == "__main__":

bitbake-dev/lib/bb/data_smart.py

@@ -32,7 +32,6 @@ import copy, os, re, sys, time, types
 import bb
 from bb   import utils, methodpool
 from COW  import COWDictBase
-from sets import Set
 from new  import classobj
 
 
@@ -142,7 +141,7 @@ class DataSmart:
             try:
                 self._special_values[keyword].add( base )
             except:
-                self._special_values[keyword] = Set()
+                self._special_values[keyword] = set()
                 self._special_values[keyword].add( base )
 
             return
@@ -154,7 +153,7 @@ class DataSmart:
         if '_' in var:
             override = var[var.rfind('_')+1:]
             if not self._seen_overrides.has_key(override):
-                self._seen_overrides[override] = Set()
+                self._seen_overrides[override] = set()
             self._seen_overrides[override].add( var )
 
         # setting var

bitbake-dev/lib/bb/fetch/__init__.py

@@ -24,16 +24,11 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re, fcntl
+import os, re
 import bb
 from   bb import data
 from   bb import persist_data
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
 class FetchError(Exception):
     """Exception raised when a download fails"""
 
@@ -65,7 +60,6 @@ def uri_replace(uri, uri_find, uri_replace, d):
         result_decoded[loc] = uri_decoded[loc]
         import types
         if type(i) == types.StringType:
-            import re
             if (re.match(i, uri_decoded[loc])):
                 result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
                 if uri_find_decoded.index(i) == 2:

bitbake-dev/lib/bb/fetch/bzr.py

@@ -29,7 +29,6 @@ import bb
 from bb import data
 from bb.fetch import Fetch
 from bb.fetch import FetchError
-from bb.fetch import MissingParameterError
 from bb.fetch import runfetchcmd
 
 class Bzr(Fetch):

bitbake-dev/lib/bb/fetch/cvs.py

@@ -26,7 +26,7 @@ BitBake build tools.
 #Based on functions from the base bb module, Copyright 2003 Holger Schurig
 #
 
-import os, re
+import os
 import bb
 from   bb import data
 from   bb.fetch import Fetch

bitbake-dev/lib/bb/fetch/git.py

@@ -20,11 +20,10 @@ BitBake 'Fetch' git implementation
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import os, re
+import os
 import bb
 from   bb    import data
 from   bb.fetch import Fetch
-from   bb.fetch import FetchError
 from   bb.fetch import runfetchcmd
 
 class Git(Fetch):
@@ -37,9 +36,12 @@ class Git(Fetch):
 
     def localpath(self, url, ud, d):
 
-        ud.proto = "rsync"
         if 'protocol' in ud.parm:
             ud.proto = ud.parm['protocol']
+        elif not ud.host:
+            ud.proto = 'file'
+        else:
+            ud.proto = "rsync"
 
         ud.branch = ud.parm.get("branch", "master")
 
@@ -49,10 +51,7 @@ class Git(Fetch):
         elif tag:
             ud.tag = tag
 
-        if not ud.tag:
+        if not ud.tag or ud.tag == "master":
-            ud.tag = self.latest_revision(url, ud, d)	
-
-        if ud.tag == "master":
             ud.tag = self.latest_revision(url, ud, d)	
 
         ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.tag), d)
@@ -90,11 +89,12 @@ class Git(Fetch):
 
         os.chdir(repodir)
         # Remove all but the .git directory
-        runfetchcmd("rm * -Rf", d)
+        if not self._contains_ref(ud.tag, d):
-        runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
+            runfetchcmd("rm * -Rf", d)
-        runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
+            runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
-        runfetchcmd("git prune-packed", d)
+            runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
-        runfetchcmd("git pack-redundant --all | xargs -r rm", d)
+            runfetchcmd("git prune-packed", d)
+            runfetchcmd("git pack-redundant --all | xargs -r rm", d)
 
         os.chdir(repodir)
         mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
@@ -120,6 +120,10 @@ class Git(Fetch):
     def suppports_srcrev(self):
         return True
 
+    def _contains_ref(self, tag, d):
+        output = runfetchcmd("git log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % tag, d, quiet=True)
+        return output.split()[0] != "0"
+
     def _revision_key(self, url, ud, d):
         """
         Return a unique key for the url

bitbake-dev/lib/bb/fetch/hg.py

@@ -24,7 +24,7 @@ BitBake 'Fetch' implementation for mercurial DRCS (hg).
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re
+import os
 import sys
 import bb
 from bb import data
@@ -123,9 +123,6 @@ class Hg(Fetch):
             bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
             runfetchcmd(updatecmd, d)
 
-            updatecmd = self._buildhgcommand(ud, d, "update")
-            bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
-            runfetchcmd(updatecmd, d)
         else:
             fetchcmd = self._buildhgcommand(ud, d, "fetch")
             bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
@@ -135,6 +132,12 @@ class Hg(Fetch):
             bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd)
             runfetchcmd(fetchcmd, d)
 	
+	# Even when we clone (fetch), we still need to update as hg's clone
+	# won't checkout the specified revision if its on a branch
+        updatecmd = self._buildhgcommand(ud, d, "update")
+        bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
+        runfetchcmd(updatecmd, d)
+
         os.chdir(ud.pkgdir)
         try:
             runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)

bitbake-dev/lib/bb/fetch/local.py

@@ -25,7 +25,7 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re
+import os
 import bb
 from   bb import data
 from   bb.fetch import Fetch

bitbake-dev/lib/bb/fetch/perforce.py

@@ -25,12 +25,11 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re
+import os
 import bb
 from   bb import data
 from   bb.fetch import Fetch
 from   bb.fetch import FetchError
-from   bb.fetch import MissingParameterError
 
 class Perforce(Fetch):
     def supports(self, url, ud, d):

bitbake-dev/lib/bb/fetch/ssh.py

@@ -37,11 +37,9 @@ IETF secsh internet draft:
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import re, os
-import bb
 from   bb import data
 from   bb.fetch import Fetch
 from   bb.fetch import FetchError
-from   bb.fetch import MissingParameterError
 
 
 __pattern__ = re.compile(r'''

bitbake-dev/lib/bb/fetch/svk.py

@@ -25,7 +25,7 @@ This implementation is for svk. It is based on the svn implementation
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re
+import os
 import bb
 from   bb import data
 from   bb.fetch import Fetch
@@ -67,6 +67,7 @@ class Svk(Fetch):
 
         svkroot = ud.host + ud.path
 
+        # pyflakes claims date is not known... it looks right
         svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module)
 
         if ud.revision:

bitbake-dev/lib/bb/fetch/svn.py

@@ -23,7 +23,7 @@ BitBake 'Fetch' implementation for svn.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re
+import os
 import sys
 import bb
 from   bb import data

bitbake-dev/lib/bb/fetch/wget.py

@@ -25,7 +25,7 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import os, re
+import os
 import bb
 from   bb import data
 from   bb.fetch import Fetch

bitbake-dev/lib/bb/msg.py

@@ -22,8 +22,8 @@ Message handling infrastructure for bitbake
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import sys, os, re, bb
+import sys, bb
-from bb import utils, event
+from bb import event
 
 debug_level = {}
 

bitbake-dev/lib/bb/providers.py

@@ -21,7 +21,7 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-import os, re
+import re
 from bb import data, utils
 import bb
 

bitbake-dev/lib/bb/runqueue.py

@@ -23,7 +23,6 @@ Handles preparation and execution of a queue of tasks
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from bb import msg, data, event, mkdirhier, utils
-from sets import Set 
 import bb, os, sys
 import signal
 import stat
@@ -544,8 +543,8 @@ class RunQueue:
 
             self.runq_fnid.append(taskData.tasks_fnid[task])
             self.runq_task.append(taskData.tasks_name[task])
-            self.runq_depends.append(Set(depends))
+            self.runq_depends.append(set(depends))
-            self.runq_revdeps.append(Set())
+            self.runq_revdeps.append(set())
 
             runq_build.append(0)
 
@@ -641,7 +640,7 @@ class RunQueue:
                 if maps[origdep] == -1:
                     bb.msg.fatal(bb.msg.domain.RunQueue, "Invalid mapping - Should never happen!")
                 newdeps.append(maps[origdep])
-            self.runq_depends[listid] = Set(newdeps)
+            self.runq_depends[listid] = set(newdeps)
 
         bb.msg.note(2, bb.msg.domain.RunQueue, "Assign Weightings")
 

bitbake-dev/lib/bb/shell.py

@@ -510,7 +510,7 @@ SRC_URI = ""
 
     def stage( self, params ):
         """Execute 'stage' on a providee"""
-        self.build( params, "stage" )
+        self.build( params, "populate_staging" )
     stage.usage = "<providee>"
 
     def status( self, params ):

bitbake-dev/lib/bb/taskdata.py

@@ -23,8 +23,7 @@ Task data collection and handling
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-from bb import data, event, mkdirhier, utils
+import bb
-import bb, os
 
 class TaskData:
     """

bitbake-dev/lib/bb/ui/crumbs/buildmanager.py

@@ -20,9 +20,7 @@
 
 import gtk
 import gobject
-import gtk.glade
 import threading
-import urllib2
 import os
 import datetime
 import time

bitbake-dev/lib/bb/ui/crumbs/runningbuild.py

@@ -20,7 +20,6 @@
 
 import gtk
 import gobject
-import gtk.glade
 
 class RunningBuildModel (gtk.TreeStore):
     (COL_TYPE, COL_PACKAGE, COL_TASK, COL_MESSAGE, COL_ICON, COL_ACTIVE) = (0, 1, 2, 3, 4, 5)

bitbake-dev/lib/bb/ui/depexp.py

@@ -20,6 +20,7 @@
 import gobject
 import gtk
 import threading
+import xmlrpclib
 
 # Package Model
 (COL_PKG_NAME) = (0)

bitbake-dev/lib/bb/ui/goggle.py

@@ -20,8 +20,7 @@
 
 import gobject
 import gtk
-import threading
+import xmlrpclib
-import bb.ui.uihelper
 from bb.ui.crumbs.runningbuild import RunningBuildTreeView, RunningBuild
 
 def event_handle_idle_func (eventHandler, build):

bitbake-dev/lib/bb/ui/knotty.py

@@ -19,11 +19,8 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
-import bb
-from bb import cooker
 
 import sys
-import time
 import itertools
 import xmlrpclib
 

bitbake-dev/lib/bb/ui/ncurses.py

@@ -44,9 +44,9 @@
 
 """
 
-import os, sys, curses, time, random, threading, itertools, time
+import os, sys, curses, itertools, time
-from curses.textpad import Textbox
 import bb
+import xmlrpclib
 from bb import ui
 from bb.ui import uihelper
 
@@ -180,6 +180,7 @@ class NCursesUI:
         def __init__( self, x, y, width, height ):
             NCursesUI.Window.__init__( self, x, y, width, height )
 
+# put that to the top again from curses.textpad import Textbox
 #            self.textbox = Textbox( self.win )
 #            t = threading.Thread()
 #            t.run = self.textbox.edit

bitbake-dev/lib/bb/ui/puccho.py

@@ -24,7 +24,6 @@ import gtk.glade
 import threading
 import urllib2
 import os
-import datetime
 
 from bb.ui.crumbs.buildmanager import BuildManager, BuildConfiguration
 from bb.ui.crumbs.buildmanager import BuildManagerTreeView

bitbake-dev/lib/bb/ui/uievent.py

@@ -24,7 +24,7 @@ server and queue them for the UI to process. This process must be used to avoid
 client/server deadlocks.
 """
 
-import sys, socket, threading
+import socket, threading
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
 
 class BBUIEventQueue:

bitbake-dev/lib/bb/xmlrpcserver.py

@@ -37,7 +37,7 @@ import xmlrpclib
 DEBUG = False
 
 from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import os, sys, inspect, select
+import inspect, select
 
 class BitBakeServerCommands():
     def __init__(self, server, cooker):