bitbake: cooker/hashserv: Allow autostarting of a local hash server using BB_HASHSERVE

Its useful, particularly in the local developer model of usage, for
bitbake to start and stop a hash equivalence server on local port,
rather than relying on one being started by the user before the build.

The new BB_HASHSERVE variable supports this.

The database handling is moved internally into the hashserv code so that
different threads/processes can be used for the server without errors.

(Bitbake rev: a4fa8f1bd88995ae60e10430316fbed63d478587)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-hashserv

@@ -41,9 +41,7 @@ def main():
     console.setLevel(level)
     logger.addHandler(console)
 
-    db = sqlite3.connect(args.database)
+    server = hashserv.create_server((args.address, args.port), args.database, args.prefix)
-
-    server = hashserv.create_server((args.address, args.port), db, args.prefix)
     server.serve_forever()
     return 0
 

bitbake/bin/bitbake-worker

@@ -418,6 +418,7 @@ class BitbakeWorker(object):
         bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"]
         for mc in self.databuilder.mcdata:
             self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
+            self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservport"])
 
     def handle_newtaskhashes(self, data):
         self.workerdata["newhashes"] = pickle.loads(data)

bitbake/lib/bb/cooker.py

@@ -31,6 +31,7 @@ import pyinotify
 import json
 import pickle
 import codecs
+import hashserv
 
 logger      = logging.getLogger("BitBake")
 collectlog  = logging.getLogger("BitBake.Collection")
@@ -230,6 +231,8 @@ class BBCooker:
         self.state = state.initial
 
         self.parser = None
+        self.hashserv = None
+        self.hashservport = None
 
         signal.signal(signal.SIGTERM, self.sigterm_exception)
         # Let SIGHUP exit as SIGTERM
@@ -368,6 +371,15 @@ class BBCooker:
 
         self.data.setVar('BB_CMDLINE', self.ui_cmdline)
 
+        if self.data.getVar("BB_HASHSERVE") == "localhost:0":
+            dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+            self.hashserv = hashserv.create_server(('localhost', 0), dbfile, '')
+            self.hashservport = "localhost:" + str(self.hashserv.server_port)
+            thread = threading.Thread(target=self.hashserv.serve_forever)
+            thread.daemon = True
+            thread.start()
+            self.data.setVar("BB_HASHSERVE", self.hashservport)
+
         #
         # Copy of the data store which has been expanded.
         # Used for firing events and accessing variables where expansion needs to be accounted for
@@ -1645,9 +1657,10 @@ class BBCooker:
 
     def post_serve(self):
         prserv.serv.auto_shutdown()
+        if self.hashserv:
+            self.hashserv.shutdown()
         bb.event.fire(CookerExit(), self.data)
 
-
     def shutdown(self, force = False):
         if force:
             self.state = state.forceshutdown

bitbake/lib/bb/runqueue.py

@@ -1254,6 +1254,7 @@ class RunQueue:
             "buildname" : self.cfgData.getVar("BUILDNAME"),
             "date" : self.cfgData.getVar("DATE"),
             "time" : self.cfgData.getVar("TIME"),
+            "hashservport" : self.cooker.hashservport,
         }
 
         worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")

bitbake/lib/hashserv/__init__.py

@@ -18,8 +18,15 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
     def log_message(self, f, *args):
         logger.debug(f, *args)
 
+    def opendb(self):
+        self.db = sqlite3.connect(self.dbname)
+        self.db.row_factory = sqlite3.Row
+
     def do_GET(self):
         try:
+            if not self.db:
+                self.opendb()
+
             p = urllib.parse.urlparse(self.path)
 
             if p.path != self.prefix + '/v1/equivalent':
@@ -52,6 +59,9 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
 
     def do_POST(self):
         try:
+            if not self.db:
+                self.opendb()
+
             p = urllib.parse.urlparse(self.path)
 
             if p.path != self.prefix + '/v1/equivalent':
@@ -123,14 +133,17 @@ class HashEquivalenceServer(BaseHTTPRequestHandler):
             self.send_error(400, explain=traceback.format_exc())
             return
 
-def create_server(addr, db, prefix=''):
+def create_server(addr, dbname, prefix=''):
     class Handler(HashEquivalenceServer):
         pass
 
-    Handler.prefix = prefix
+    db = sqlite3.connect(dbname)
-    Handler.db = db
     db.row_factory = sqlite3.Row
 
+    Handler.prefix = prefix
+    Handler.db = None
+    Handler.dbname = dbname
+
     with contextlib.closing(db.cursor()) as cursor:
         cursor.execute('''
             CREATE TABLE IF NOT EXISTS tasks_v2 (

bitbake/lib/hashserv/tests.py

@@ -11,14 +11,15 @@ import sqlite3
 import hashlib
 import urllib.request
 import json
+import tempfile
 from . import create_server
 
 class TestHashEquivalenceServer(unittest.TestCase):
     def setUp(self):
-        # Start an in memory hash equivalence server in the background bound to
+        # Start a hash equivalence server in the background bound to
         # an ephemeral port
-        db = sqlite3.connect(':memory:', check_same_thread=False)
+        self.dbfile = tempfile.NamedTemporaryFile(prefix="bb-hashserv-db-")
-        self.server = create_server(('localhost', 0), db)
+        self.server = create_server(('localhost', 0), self.dbfile.name)
         self.server_addr = 'http://localhost:%d' % self.server.socket.getsockname()[1]
         self.server_thread = threading.Thread(target=self.server.serve_forever)
         self.server_thread.start()