recipetool/create_go: proxy module fetching to go-mod-update-modules

Now that the go-mod-update-modules class exists, this Go handler can
create a stub recipe and then proxy the module handling to the class.

(From OE-Core rev: 0aa406d0582d32399c48dfa78f24adc75696112c)

Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

meta/classes-recipe/go-mod-update-modules.bbclass

@@ -48,7 +48,7 @@ python do_update_modules() {
 
         env = dict(os.environ, GOMODCACHE=mod_cache_dir)
 
-        source = d.expand("${WORKDIR}/${GO_SRCURI_DESTSUFFIX}")
+        source = d.expand("${UNPACKDIR}/${GO_SRCURI_DESTSUFFIX}")
         output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True)
         go_mod = json.loads(output)
 
@@ -78,7 +78,7 @@ python do_update_modules() {
 
             mod_dir = mod['Dir']
 
-            if mod_dir.startswith(s_dir):
+            if not mod_dir.startswith(mod_cache_dir):
                 continue
 
             path = os.path.relpath(mod_dir, mod_cache_dir)

scripts/lib/recipetool/create_go.py

@@ -11,17 +11,15 @@
 
 
 from recipetool.create import RecipeHandler, handle_license_vars
-from recipetool.create import find_licenses
 
 import bb.utils
 import json
 import logging
 import os
 import re
+import subprocess
 import sys
 import tempfile
-import urllib.parse
-import urllib.request
 
 
 logger = logging.getLogger('recipetool')
@@ -66,97 +64,6 @@ class GoRecipeHandler(RecipeHandler):
 
         return bindir
 
-    @staticmethod
-    def __unescape_path(path):
-        """Unescape capital letters using exclamation points."""
-        return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path)
-
-    @staticmethod
-    def __fold_uri(uri):
-        """Fold URI for sorting shorter module paths before longer."""
-        return uri.replace(';', ' ').replace('/', '!')
-
-    @staticmethod
-    def __go_run_cmd(cmd, cwd, d):
-        env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE'))
-        return bb.process.run(cmd, env=env, shell=True, cwd=cwd)
-
-    def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d):
-        moddir = d.getVar('GOMODCACHE')
-
-        # List main packages and their dependencies with the go list command.
-        stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d)
-        pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']')
-
-        # Collect licenses for the dependencies.
-        licenses = set()
-        lic_files_chksum = []
-        lic_files = {}
-        for pkg in pkgs:
-            # TODO: If the package is in a subdirectory with its own license
-            # files then report those istead of the license files found in the
-            # module root directory.
-            mod = pkg.get('Module', None)
-            if not mod or mod.get('Main', False):
-                continue
-            path = os.path.relpath(mod['Dir'], moddir)
-            for lic in find_licenses(mod['Dir'], d):
-                lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2])
-
-        for lic_file in lic_files:
-            licenses.add(lic_files[lic_file][0])
-            lic_files_chksum.append(
-                f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}')
-
-        # Collect the module cache files downloaded by the go list command as
-        # the go list command knows best what the go list command needs and it
-        # needs more files in the module cache than the go install command as
-        # it doesn't do the dependency pruning mentioned in the Go module
-        # reference, https://go.dev/ref/mod, for go 1.17 or higher.
-        src_uris = []
-        downloaddir = os.path.join(moddir, 'cache', 'download')
-        for dirpath, _, filenames in os.walk(downloaddir):
-            path, base = os.path.split(os.path.relpath(dirpath, downloaddir))
-            if base != '@v':
-                continue
-            path = self.__unescape_path(path)
-            zipver = None
-            for name in filenames:
-                ver, ext = os.path.splitext(name)
-                if ext == '.zip':
-                    chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
-                    src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}')
-                    zipver = ver
-                    break
-            for name in filenames:
-                ver, ext = os.path.splitext(name)
-                if ext == '.mod' and ver != zipver:
-                    chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
-                    src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}')
-
-        self.__go_run_cmd("go clean -modcache", srctree, d)
-
-        licenses_basename = "{pn}-licenses.inc"
-        licenses_filename = os.path.join(localfilesdir, licenses_basename)
-        with open(licenses_filename, "w") as f:
-            f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n')
-            f.write('LIC_FILES_CHKSUM += "\\\n')
-            for lic in sorted(lic_files_chksum, key=self.__fold_uri):
-                f.write('    ' + lic + ' \\\n')
-            f.write('"\n')
-
-        extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename
-
-        go_mods_basename = "{pn}-go-mods.inc"
-        go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
-        with open(go_mods_filename, "w") as f:
-            f.write('SRC_URI += "\\\n')
-            for uri in sorted(src_uris, key=self.__fold_uri):
-                f.write('    ' + uri + ' \\\n')
-            f.write('"\n')
-
-        extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename
-
     def process(self, srctree, classes, lines_before,
                 lines_after, handled, extravalues):
 
@@ -167,37 +74,52 @@ class GoRecipeHandler(RecipeHandler):
         if not files:
             return False
 
-        d = bb.data.createCopy(tinfoil.config_data)
         go_bindir = self.__ensure_go()
         if not go_bindir:
             sys.exit(14)
 
-        d.prependVar('PATH', '%s:' % go_bindir)
         handled.append('buildsystem')
         classes.append("go-mod")
 
-        tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-')
+        # Use go-mod-update-modules to set the full SRC_URI and LICENSE
-        d.setVar('GOMODCACHE', tmp_mod_dir)
+        classes.append("go-mod-update-modules")
+        extravalues["run_tasks"] = "update_modules"
 
-        stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
+        with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir:
-        go_mod = json.loads(stdout)
+            env = dict(os.environ)
-        go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
+            env["PATH"] += f":{go_bindir}"
+            env['GOMODCACHE'] = tmp_mod_dir
 
-        localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+            stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True)
-        extravalues.setdefault('extrafiles', {})
+            go_mod = json.loads(stdout)
+            go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
 
-        # Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
+            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
-        self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d)
+            extravalues.setdefault('extrafiles', {})
 
-        # Do generic license handling
+            # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
-        handle_license_vars(srctree, lines_before, handled, extravalues, d)
+            basename = "{pn}-licenses.inc"
-        self.__rewrite_lic_vars(lines_before)
+            filename = os.path.join(localfilesdir, basename)
+            with open(filename, "w") as f:
+                f.write("# FROM RECIPETOOL\n")
+            extravalues['extrafiles'][f"../{basename}"] = filename
 
-        self.__rewrite_src_uri(lines_before)
+            basename = "{pn}-go-mods.inc"
+            filename = os.path.join(localfilesdir, basename)
+            with open(filename, "w") as f:
+                f.write("# FROM RECIPETOOL\n")
+            extravalues['extrafiles'][f"../{basename}"] = filename
 
-        lines_before.append('require ${BPN}-licenses.inc')
+            # Do generic license handling
-        lines_before.append('require ${BPN}-go-mods.inc')
+            d = bb.data.createCopy(tinfoil.config_data)
-        lines_before.append(f'GO_IMPORT = "{go_import}"')
+            handle_license_vars(srctree, lines_before, handled, extravalues, d)
+            self.__rewrite_lic_vars(lines_before)
+
+            self.__rewrite_src_uri(lines_before)
+
+            lines_before.append('require ${BPN}-licenses.inc')
+            lines_before.append('require ${BPN}-go-mods.inc')
+            lines_before.append(f'GO_IMPORT = "{go_import}"')
 
     def __update_lines_before(self, updated, newlines, lines_before):
         if updated:
@@ -210,10 +132,8 @@ class GoRecipeHandler(RecipeHandler):
         return updated
 
     def __rewrite_lic_vars(self, lines_before):
-
         def varfunc(varname, origvalue, op, newlines):
-            if varname == 'LICENSE':
+            import urllib.parse
-                return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True
             if varname == 'LIC_FILES_CHKSUM':
                 new_licenses = []
                 licenses = origvalue.split('\\')
@@ -235,7 +155,7 @@ class GoRecipeHandler(RecipeHandler):
             return origvalue, None, 0, True
 
         updated, newlines = bb.utils.edit_metadata(
-            lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
+            lines_before, ['LIC_FILES_CHKSUM'], varfunc)
         return self.__update_lines_before(updated, newlines, lines_before)
 
     def __rewrite_src_uri(self, lines_before):