run-jinja-parser: Add converter Jinja2 template to YAML parser

run-jinja-parser converts the Jinja2 template from lava-templates folder into YAML pipeline job configuration used in LAVA. Jinja2 provides a standard template to be modify/update to meet other architecture supports. The lava-template/generate-jobconfig.jinja2 are to be couple with the JSON config-intelqa-x86_64-lava.json when defining your architecture and hardware configuration at LAVA end. Signed-off-by: Aaron Chan <aaron.chun.yew.chan@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2025-10-22 23:02:21 +02:00 · 2018-08-29 21:25:51 +08:00 · 2018-08-29 21:25:51 +08:00 · fc933b86e5
commit fc933b86e5
parent a68fd9beab
1 changed files with 96 additions and 0 deletions
--- a/lava/run-jinja-parser
+++ b/lava/run-jinja-parser
@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+#
+# Parser loads JSON file (e.g. config-intelqa-x86_64-lava) and converts Jinja2 template 
+# format into a LAVA Job configuration in YAML format.
+#
+# Parameters:
+# $1 - Define the absolute path of Jinja2 template stored
+# $2 - Inherits the Job name in autobuilder (e.g. nightly-x86)
+# $3 - Inherits the Build number in autobuilder (Defaults to None)
+# $4 - Device type definition on LAVA Dispatcher
+#
+import os
+import sys
+import re
+import json
+import jinja2
+import time
+from jinja2 import Template, Environment, FileSystemLoader
+
+sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),"scripts"))
+import utils
+
+# Enable this section on manual run
+# os.environ["ABHELPER_JSON"]="config.json /home/ab/yocto-autobuilder-helper/config-intelqa-x86_64-lava.json"
+
+def jinja_helper():
+    print("USAGE: python3 run-jinja-parser <Jinja Template> <Build Name> <Build num> <LAVA Device Type>")
+    print("       python3 scripts/run-jinja-parser lava/device/bsp-packages.jinja2 nightly-x86-64-bsp None minnowboard")
+    sys.exit(0)
+
+# Create Job definition in YAML based on autobuilder job name
+def jinja_writer(name, data):
+    yamlFile=name + ".yaml"
+    yamlNewFile= "-".join([name, time.strftime("%d%m%Y-%H%M%S")]) + ".yaml"
+    if os.path.isfile(yamlFile):
+        os.rename(yamlFile, yamlNewFile)
+        print("INFO: Found previous job config [%s] & rename to [%s]" % (yamlFile, yamlNewFile))
+    with open(yamlFile, "w+") as fh:
+        fh.write(data)
+    fh.close()
+
+# Handles data expansion based on pattern matching
+def getconfig_expand(config, ourconfig, pattern, match, buildname, buildnum):
+    newconfig={}
+    expansion=re.compile(pattern)
+    for items in ourconfig.items():
+        if items[0] in match:
+            if items[0] in "DEPLOY_DIR":
+                imagedeploy= "file://" + os.path.join(items[1], buildname)
+                newconfig[items[0]] = imagedeploy
+                if buildnum is not None and buildnum != "None":
+                    newconfig[items[0]] = os.path.join(imagedeploy, str(buildnum))
+            else:
+                newconfig[items[0]] = items[1]
+            config=config.replace('${' + items[0] + '}', newconfig[items[0]])
+    newconfig['DEPLOY_DIR_IMAGE']=config
+    return newconfig['DEPLOY_DIR_IMAGE']
+
+try:
+    jinjaTempl=sys.argv[1]
+    target=sys.argv[2]
+    buildnum=sys.argv[3]
+    device=sys.argv[4]
+    debug=True
+except:
+    jinja_helper()
+
+ourconfig  = utils.loadconfig()
+jobconfig  = ourconfig['overrides'][target]
+lavaconfig = ourconfig['lava-devices'][device]
+deploydir  = getconfig_expand(jobconfig['DEPLOY_DIR_IMAGE'], jobconfig, "\${(.+)}/images/\${(.+)}/", ['DEPLOY_DIR', 'MACHINE'], target, buildnum)
+newconfig  = { 'DEPLOY_DIR_IMAGE' : deploydir }
+jinjaTempl = os.path.abspath(jinjaTempl)
+
+for img in ['kernel', 'modules', 'nfsrootfs']:
+    lavaconfig['deploy'][img]['url'] = getconfig_expand(lavaconfig['deploy'][img]['url'], newconfig, "\${(.+)}.+", ['DEPLOY_DIR_IMAGE'], target, buildnum)
+lavaconfig['device_type'] = device
+
+if not os.path.isfile(jinjaTempl):
+    print("ERROR: Unable to find Jinja2 Template: [%s]" % jinjaTempl)
+    sys.exit(1)
+
+# JSON Dumps
+if debug:
+    print(json.dumps(lavaconfig, indent=4))
+
+jinjaPath = "/".join(jinjaTempl.split("/")[0:-1])
+jinjaFile = jinjaTempl.split("/")[-1]
+
+templateLoader = jinja2.FileSystemLoader(searchpath=jinjaPath)
+templateEnv    = jinja2.Environment(loader=templateLoader)
+templateJinja  = templateEnv.get_template(jinjaFile)
+outText        = templateJinja.render(lavaconfig)
+
+jinja_writer(target, outText)
+print("INFO: Job configuration [%s] is ready to be triggered in next step" % target)