package: Call file to determine elf status in parallel

This allows the calls to is_elf (which calls file) to happen in parallel
allowing a speedup of do_package and do_populate_sysroot for native
recipes.

(From OE-Core rev: bbe0d3e26484f3f347262d40a8a9d415ce21fb43)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/classes/package.bbclass

@@ -949,6 +949,8 @@ python split_and_strip_files () {
     skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split()
     if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \
             d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
+        checkelf = {}
+        checkelflinks = {}
         for root, dirs, files in cpath.walk(dvar):
             for f in files:
                 file = os.path.join(root, f)
@@ -982,44 +984,57 @@ python split_and_strip_files () {
                 # Check its an executable
                 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
                         or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
-                    # If it's a symlink, and points to an ELF file, we capture the readlink target
+
                     if cpath.islink(file):
-                        target = os.readlink(file)
+                        checkelflinks[file] = ltarget
-                        if oe.package.is_elf(ltarget):
-                            #bb.note("Sym: %s (%d)" % (ltarget, oe.package.is_elf(ltarget)))
-                            symlinks[file] = target
                         continue
+                    # Use a reference of device ID and inode number to identify files
+                    file_reference = "%d_%d" % (s.st_dev, s.st_ino)
+                    checkelf[file] = (file, file_reference)
 
-                    # It's a file (or hardlink), not a link
+        results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d)
-                    # ...but is it ELF, and is it already stripped?
+        results_map = {}
-                    elf_file = oe.package.is_elf(file)
+        for (ltarget, elf_file) in results:
-                    if elf_file & 1:
+            results_map[ltarget] = elf_file
-                        if elf_file & 2:
+        for file in checkelflinks:
-                            if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split():
+            ltarget = checkelflinks[file]
-                                bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
+            # If it's a symlink, and points to an ELF file, we capture the readlink target
-                            else:
+            if results_map[ltarget]:
-                                msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
+                target = os.readlink(file)
-                                package_qa_handle_error("already-stripped", msg, d)
+                #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget]))
-                            continue
+                symlinks[file] = target
 
-                        # At this point we have an unstripped elf file. We need to:
+        results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d)
-                        #  a) Make sure any file we strip is not hardlinked to anything else outside this tree
+        for (file, elf_file) in results:
-                        #  b) Only strip any hardlinked file once (no races)
+            # It's a file (or hardlink), not a link
-                        #  c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
+            # ...but is it ELF, and is it already stripped?
+            if elf_file & 1:
+                if elf_file & 2:
+                    if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split():
+                        bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
+                    else:
+                        msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
+                        package_qa_handle_error("already-stripped", msg, d)
+                    continue
 
-                        # Use a reference of device ID and inode number to identify files
+                # At this point we have an unstripped elf file. We need to:
-                        file_reference = "%d_%d" % (s.st_dev, s.st_ino)
+                #  a) Make sure any file we strip is not hardlinked to anything else outside this tree
-                        if file_reference in inodes:
+                #  b) Only strip any hardlinked file once (no races)
-                            os.unlink(file)
+                #  c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
-                            os.link(inodes[file_reference][0], file)
+
-                            inodes[file_reference].append(file)
+                # Use a reference of device ID and inode number to identify files
-                        else:
+                file_reference = checkelf[file]
-                            inodes[file_reference] = [file]
+                if file_reference in inodes:
-                            # break hardlink
+                    os.unlink(file)
-                            bb.utils.copyfile(file, file)
+                    os.link(inodes[file_reference][0], file)
-                            elffiles[file] = elf_file
+                    inodes[file_reference].append(file)
-                        # Modified the file so clear the cache
+                else:
-                        cpath.updatecache(file)
+                    inodes[file_reference] = [file]
+                    # break hardlink
+                    bb.utils.copyfile(file, file)
+                    elffiles[file] = elf_file
+                # Modified the file so clear the cache
+                cpath.updatecache(file)
 
     #
     # First lets process debug splitting

meta/classes/staging.bbclass

@@ -70,7 +70,7 @@ sysroot_stage_all() {
 python sysroot_strip () {
     inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP')
     if inhibit_sysroot and oe.types.boolean(inhibit_sysroot):
-        return 0
+        return
 
     dstdir = d.getVar('SYSROOT_DESTDIR')
     pn = d.getVar('PN')
@@ -79,7 +79,7 @@ python sysroot_strip () {
     qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split()
     strip_cmd = d.getVar("STRIP")
 
-    oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir,
+    oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d,
                            qa_already_stripped=qa_already_stripped)
 }
 

meta/lib/oe/package.py

@@ -74,7 +74,7 @@ def is_elf(path):
         if "relocatable" in result:
             if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path):
                 exec_type |= 16
-    return exec_type
+    return (path, exec_type)
 
 def is_static_lib(path):
     if path.endswith('.a') and not os.path.islink(path):
@@ -86,7 +86,7 @@ def is_static_lib(path):
             return start == magic
     return False
 
-def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=False):
+def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=False):
     """
     Strip executable code (like executables, shared libraries) _in_place_
     - Based on sysroot_strip in staging.bbclass
@@ -107,6 +107,8 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
     #
     # First lets figure out all of the files we may have to process
     #
+    checkelf = []
+    inodecache = {}
     for root, dirs, files in os.walk(dstdir):
         for f in files:
             file = os.path.join(root, f)
@@ -132,7 +134,11 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
 
                 # It's a file (or hardlink), not a link
                 # ...but is it ELF, and is it already stripped?
-                elf_file = is_elf(file)
+                checkelf.append(file)
+                inodecache[file] = s.st_ino
+    results = oe.utils.multiprocess_launch(is_elf, checkelf, d)
+    for (file, elf_file) in results:
+                #elf_file = is_elf(file)
                 if elf_file & 1:
                     if elf_file & 2:
                         if qa_already_stripped:
@@ -141,12 +147,12 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
                             bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn))
                         continue
 
-                    if s.st_ino in inodes:
+                    if inodecache[file] in inodes:
                         os.unlink(file)
-                        os.link(inodes[s.st_ino], file)
+                        os.link(inodes[inodecache[file]], file)
                     else:
                         # break hardlinks so that we do not strip the original.
-                        inodes[s.st_ino] = file
+                        inodes[inodecache[file]] = file
                         bb.utils.copyfile(file, file)
                         elffiles[file] = elf_file