verify-bashisms: check scripts only once, include original file and line

Several scripts that are defined in .bbclass files end up in multiple
different recipes. It's better (faster, less repetitive error reports)
to check them only once.

In addition, the real information for the developer is where he can
find the script, not which recipe file uses it. verify-bashisms now
prints the original file instead of the recipe whenever possible
(i.e. 'filename' is set) and also bumps the line number so that it is
relative to the file and not the script.

Example with one real error and one added just for testing:

  $ verify-bashisms core-image-minimal core-image-sato
  Loading cache: 100% |#################################################################################| Time: 0:00:00
  Loaded 2935 entries from dependency cache.
  Parsing recipes: 100% |###############################################################################| Time: 0:00:01
  Parsing of 2137 .bb files complete (2101 cached, 36 parsed). 2935 targets, 412 skipped, 0 masked, 0 errors.
  Generating scripts...
  Scanning scripts...

  /.../openembedded-core/meta/classes/populate_sdk_ext.bbclass
   possible bashism in install_tools line 515 (should be 'b = a'):
  	if [ "${SDK_INCLUDE_TOOLCHAIN}" == "1" -a ! -e $unfsd_path ] ; then
   possible bashism in install_tools line 521 (type):
            type fixme

(From OE-Core rev: ca4932b60f464430266cc43e34122b2973e8a200)

Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

scripts/verify-bashisms

@@ -22,7 +22,9 @@ def is_whitelisted(s):
             return True
     return False
 
-def process(recipe, function, script):
+SCRIPT_LINENO_RE = re.compile(r' line (\d+) ')
+
+def process(filename, function, lineno, script):
     import tempfile
 
     if not script.startswith("#!"):
@@ -45,13 +47,25 @@ def process(recipe, function, script):
             print("Unexpected output from checkbashism: %s" % str(output))
             return
 
-        # Turn the output into a list of (message, source) values
+        # Turn the output into a single string like this:
+        # /.../foobar.bb
+        #  possible bashism in updatercd_postrm line 2 (type):
+        #   if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then
+        #  ...
+        #   ...
         result = []
         # Check the results against the whitelist
         for message, source in zip(output[0::2], output[1::2]):
             if not is_whitelisted(source):
-                result.append((message, source))
+                if lineno is not None:
-        return result
+                    message = SCRIPT_LINENO_RE.sub(lambda m: ' line %d ' % (int(m.group(1)) + int(lineno) - 1),
+                                                   message)
+                result.extend([' ' + message, '  ' + source])
+        if result:
+            result.insert(0, filename)
+            return '\n'.join(result)
+        else:
+            return None
 
 def get_tinfoil():
     scripts_path = os.path.dirname(os.path.realpath(__file__))
@@ -75,12 +89,8 @@ if __name__=='__main__':
     # initializing the pool and connecting to the
     # bitbake server is crucial, don't change it.
     def func(item):
-        fn, scripts = item
+        (filename, key, lineno), script = item
-        result = []
+        return process(filename, key, lineno, script)
-        for key, script in scripts:
-            r = process(fn, key, script)
-            if r: result.extend(r)
-        return fn, result
 
     import multiprocessing
     pool = multiprocessing.Pool()
@@ -97,27 +107,33 @@ if __name__=='__main__':
     else:
         initial_pns = sorted(pkg_pn)
 
-    pns = {}
+    pns = set()
+    scripts = {}
     print("Generating scripts...")
     for pn in initial_pns:
         for fn in pkg_pn[pn]:
             # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe
+            # (at least in general - there is some risk that the variants contain different scripts)
             realfn, _, _ = bb.cache.virtualfn2realfn(fn)
             if realfn not in pns:
+                pns.add(realfn)
                 data = tinfoil.parse_recipe_file(realfn)
-                scripts = []
                 for key in data.keys():
                     if data.getVarFlag(key, "func") and not data.getVarFlag(key, "python"):
                         script = data.getVar(key, False)
                         if script:
-                            scripts.append((key, script))
+                            filename = data.getVarFlag(key, "filename")
-                pns[realfn] = scripts
+                            lineno = data.getVarFlag(key, "lineno")
+                            # There's no point in checking a function multiple
+                            # times just because different recipes include it.
+                            # We identify unique scripts by file, name, and (just in case)
+                            # line number.
+                            attributes = (filename or realfn, key, lineno)
+                            scripts.setdefault(attributes, script)
+
 
     print("Scanning scripts...\n")
-    for pn, results in pool.imap(func, pns.items()):
+    for result in pool.imap(func, scripts.items()):
-        if results:
+        if result:
-            print(pn)
+            print(result)
-            for message,source in results:
-                print(" %s\n  %s" % (message, source))
-            print()
     tinfoil.shutdown()