utils: Allow customisation using ABHELPER_JSON from the environment

Usage is documented in README and an example, local-example.json is
included.

Also clean up the parameter to loadconfig() as its actually not needed.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

README

@@ -5,6 +5,23 @@ release and contains the hooks that do the actual build configuration and execut
 pre-commit-hook.sh can be used to verify the JSON before committing, symlink this
 to .git/hooks/pre-commit (ln -s ../../scripts/pre-commit-hook.sh .git/hooks/pre-commit).
 
+Its likely most users will end up having to customise this repository for their needs. The
+scripts themselves should be more generically reusable, the config.json, less so as it 
+represents the Yocto Project Autobuilder test matrix.
+
+There are two customisation options possible, one is through variable substitution, the other
+is through overlaying configuration files. The standard config.json tries to at least allow
+substitution of the paths. A local-example.json is included to show how you could override
+these from a separate config file, simply passing:
+
+ABHELPER_JSON="config.json local-example.json"
+
+into the environment of the autobuilder.
+
+ABHELPER_JSON="config.json /some/location/local.json"
+
+would also allow customisation.
+
 Authors: 
   Richard Purdie <richard.purdie@linuxfoundation.org>
   Joshua Lock <joshua.g.lock@intel.com>

janitor/ab-janitor

@@ -23,7 +23,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'scripts'))
 
 import utils
 
-ourconfig = utils.loadconfig(os.path.dirname(__file__) + "/../scripts/dummy")
+ourconfig = utils.loadconfig()
 
 if "TRASH_DIR" not in ourconfig:
     print("Please set TRASH_DIR in the configuration file")

janitor/clobberdir

@@ -17,7 +17,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'scripts'))
 
 import utils
 
-ourconfig = utils.loadconfig(os.path.dirname(__file__) + "/../scripts/dummy")
+ourconfig = utils.loadconfig()
 
 
 def mkdir(path):

local-example.json

@@ -0,0 +1,4 @@
+{
+    "BASE_HOMEDIR" : "/home/someotheruser",
+    "BASE_SHAREDDIR" : "/some/other/shared/dir"
+}

scripts/layer-config

@@ -22,7 +22,7 @@ targetdir = sys.argv[1]
 target = sys.argv[2]
 targetbuilddir = targetdir
 
-ourconfig = utils.loadconfig(__file__)
+ourconfig = utils.loadconfig()
 
 def bitbakecmd(targetbuilddir, cmd):
     ret = subprocess.call(". ./oe-init-build-env; %s" % cmd, shell=True, cwd=targetbuilddir)

scripts/prepare-shared-repos

@@ -25,7 +25,7 @@ publish = None
 if sys.argv[3] != "None":
     publish = sys.argv[3]
 
-ourconfig = utils.loadconfig(__file__)
+ourconfig = utils.loadconfig()
 
 with open(repojson) as f:
     repos = json.load(f)

scripts/run-config

@@ -38,7 +38,7 @@ if sys.argv[8] != "None":
     errorurl = sys.argv[8]
 
 scriptsdir = os.path.dirname(os.path.realpath(__file__))
-ourconfig = utils.loadconfig(__file__)
+ourconfig = utils.loadconfig()
 
 # Find out the number of steps this target has
 maxsteps = 1

scripts/send-qa-email

@@ -31,7 +31,7 @@ if send != "True" or publish == "None" or rel_name == "None":
     sys.exit(0)
 
 scriptsdir = os.path.dirname(os.path.realpath(__file__))
-ourconfig = utils.loadconfig(__file__)
+ourconfig = utils.loadconfig()
 
 with open(repojson) as f:
     repos = json.load(f)

scripts/setup-config

@@ -31,7 +31,7 @@ reponame = sys.argv[5]
 sstate_release = sys.argv[6]
 buildappsrcrev = sys.argv[7]
 
-ourconfig = utils.loadconfig(__file__)
+ourconfig = utils.loadconfig()
 
 variables = []
 

scripts/shared-repo-unpack

@@ -33,7 +33,7 @@ if sys.argv[5] != "None":
 
 
 scriptsdir = os.path.dirname(os.path.realpath(__file__))
-ourconfig = utils.loadconfig(__file__)
+ourconfig = utils.loadconfig()
 
 stashdir = utils.getconfig("REPO_STASH_DIR", ourconfig)
 

scripts/utils.py

@@ -113,13 +113,32 @@ def expandtemplates(ourconfig):
     return ourconfig
 
 #
-# Helper to load the config.json file for scripts in the scripts directory (pass in __file__)
+# Helper to load the json config files
 #
-def loadconfig(f):
+# Defaults to the top level config.json file
-    scriptsdir = os.path.dirname(os.path.realpath(f))
+# however this can be customised from the environment, e.g.:
+#   ABHELPER_JSON="config.json local.json"
+#   ABHELPER_JSON="config.json /path/to/local.json"
+# files without paths are assumed to be in scripts/..
+#
+# Values from later files overwrite values from earlier files
+#
+def loadconfig():
+    files = "config.json"
+    if "ABHELPER_JSON" in os.environ:
+        files = os.environ["ABHELPER_JSON"]
 
-    with open(os.path.join(scriptsdir, '..', 'config.json')) as f:
+    scriptsdir = os.path.dirname(os.path.realpath(__file__))
-        ourconfig = json.load(f)
+
+    ourconfig = {}
+    for f in files.split():
+        p = f
+        if not f.startswith("/"):
+            p = os.path.join(scriptsdir, '..', f)
+        with open(p) as j:
+            config = json.load(j)
+            for c in config:
+                ourconfig[c] = config[c]
 
     # Expand templates in the configuration
     ourconfig = expandtemplates(ourconfig)