mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-04 20:54:45 +02:00
recipetool/create_go: proxy module fetching to go-mod-update-modules
Now that the go-mod-update-modules class exists, this Go handler can create a stub recipe and then proxy the module handling to the class. (From OE-Core rev: 0aa406d0582d32399c48dfa78f24adc75696112c) Signed-off-by: Ross Burton <ross.burton@arm.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Ross Burton
Richard Purdie
parent
9291f67f1e
commit
43434a79c0
|
|
@ -48,7 +48,7 @@ python do_update_modules() {
|
|||
|
||||
env = dict(os.environ, GOMODCACHE=mod_cache_dir)
|
||||
|
||||
source = d.expand("${WORKDIR}/${GO_SRCURI_DESTSUFFIX}")
|
||||
source = d.expand("${UNPACKDIR}/${GO_SRCURI_DESTSUFFIX}")
|
||||
output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True)
|
||||
go_mod = json.loads(output)
|
||||
|
||||
|
|
@ -78,7 +78,7 @@ python do_update_modules() {
|
|||
|
||||
mod_dir = mod['Dir']
|
||||
|
||||
if mod_dir.startswith(s_dir):
|
||||
if not mod_dir.startswith(mod_cache_dir):
|
||||
continue
|
||||
|
||||
path = os.path.relpath(mod_dir, mod_cache_dir)
|
||||
|
|
|
|||
|
|
@ -11,17 +11,15 @@
|
|||
|
||||
|
||||
from recipetool.create import RecipeHandler, handle_license_vars
|
||||
from recipetool.create import find_licenses
|
||||
|
||||
import bb.utils
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
logger = logging.getLogger('recipetool')
|
||||
|
|
@ -66,97 +64,6 @@ class GoRecipeHandler(RecipeHandler):
|
|||
|
||||
return bindir
|
||||
|
||||
@staticmethod
|
||||
def __unescape_path(path):
|
||||
"""Unescape capital letters using exclamation points."""
|
||||
return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path)
|
||||
|
||||
@staticmethod
|
||||
def __fold_uri(uri):
|
||||
"""Fold URI for sorting shorter module paths before longer."""
|
||||
return uri.replace(';', ' ').replace('/', '!')
|
||||
|
||||
@staticmethod
|
||||
def __go_run_cmd(cmd, cwd, d):
|
||||
env = dict(os.environ, PATH=d.getVar('PATH'), GOMODCACHE=d.getVar('GOMODCACHE'))
|
||||
return bb.process.run(cmd, env=env, shell=True, cwd=cwd)
|
||||
|
||||
def __go_mod(self, go_mod, srctree, localfilesdir, extravalues, d):
|
||||
moddir = d.getVar('GOMODCACHE')
|
||||
|
||||
# List main packages and their dependencies with the go list command.
|
||||
stdout, _ = self.__go_run_cmd(f"go list -json=Dir,Module -deps {go_mod['Module']['Path']}/...", srctree, d)
|
||||
pkgs = json.loads('[' + stdout.replace('}\n{', '},\n{') + ']')
|
||||
|
||||
# Collect licenses for the dependencies.
|
||||
licenses = set()
|
||||
lic_files_chksum = []
|
||||
lic_files = {}
|
||||
for pkg in pkgs:
|
||||
# TODO: If the package is in a subdirectory with its own license
|
||||
# files then report those istead of the license files found in the
|
||||
# module root directory.
|
||||
mod = pkg.get('Module', None)
|
||||
if not mod or mod.get('Main', False):
|
||||
continue
|
||||
path = os.path.relpath(mod['Dir'], moddir)
|
||||
for lic in find_licenses(mod['Dir'], d):
|
||||
lic_files[os.path.join(path, lic[1])] = (lic[0], lic[2])
|
||||
|
||||
for lic_file in lic_files:
|
||||
licenses.add(lic_files[lic_file][0])
|
||||
lic_files_chksum.append(
|
||||
f'file://pkg/mod/{lic_file};md5={lic_files[lic_file][1]}')
|
||||
|
||||
# Collect the module cache files downloaded by the go list command as
|
||||
# the go list command knows best what the go list command needs and it
|
||||
# needs more files in the module cache than the go install command as
|
||||
# it doesn't do the dependency pruning mentioned in the Go module
|
||||
# reference, https://go.dev/ref/mod, for go 1.17 or higher.
|
||||
src_uris = []
|
||||
downloaddir = os.path.join(moddir, 'cache', 'download')
|
||||
for dirpath, _, filenames in os.walk(downloaddir):
|
||||
path, base = os.path.split(os.path.relpath(dirpath, downloaddir))
|
||||
if base != '@v':
|
||||
continue
|
||||
path = self.__unescape_path(path)
|
||||
zipver = None
|
||||
for name in filenames:
|
||||
ver, ext = os.path.splitext(name)
|
||||
if ext == '.zip':
|
||||
chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
|
||||
src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}')
|
||||
zipver = ver
|
||||
break
|
||||
for name in filenames:
|
||||
ver, ext = os.path.splitext(name)
|
||||
if ext == '.mod' and ver != zipver:
|
||||
chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
|
||||
src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}')
|
||||
|
||||
self.__go_run_cmd("go clean -modcache", srctree, d)
|
||||
|
||||
licenses_basename = "{pn}-licenses.inc"
|
||||
licenses_filename = os.path.join(localfilesdir, licenses_basename)
|
||||
with open(licenses_filename, "w") as f:
|
||||
f.write(f'GO_MOD_LICENSES = "{" & ".join(sorted(licenses))}"\n\n')
|
||||
f.write('LIC_FILES_CHKSUM += "\\\n')
|
||||
for lic in sorted(lic_files_chksum, key=self.__fold_uri):
|
||||
f.write(' ' + lic + ' \\\n')
|
||||
f.write('"\n')
|
||||
|
||||
extravalues['extrafiles'][f"../{licenses_basename}"] = licenses_filename
|
||||
|
||||
go_mods_basename = "{pn}-go-mods.inc"
|
||||
go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
|
||||
with open(go_mods_filename, "w") as f:
|
||||
f.write('SRC_URI += "\\\n')
|
||||
for uri in sorted(src_uris, key=self.__fold_uri):
|
||||
f.write(' ' + uri + ' \\\n')
|
||||
f.write('"\n')
|
||||
|
||||
extravalues['extrafiles'][f"../{go_mods_basename}"] = go_mods_filename
|
||||
|
||||
def process(self, srctree, classes, lines_before,
|
||||
lines_after, handled, extravalues):
|
||||
|
||||
|
|
@ -167,37 +74,52 @@ class GoRecipeHandler(RecipeHandler):
|
|||
if not files:
|
||||
return False
|
||||
|
||||
d = bb.data.createCopy(tinfoil.config_data)
|
||||
go_bindir = self.__ensure_go()
|
||||
if not go_bindir:
|
||||
sys.exit(14)
|
||||
|
||||
d.prependVar('PATH', '%s:' % go_bindir)
|
||||
handled.append('buildsystem')
|
||||
classes.append("go-mod")
|
||||
|
||||
tmp_mod_dir = tempfile.mkdtemp(prefix='go-mod-')
|
||||
d.setVar('GOMODCACHE', tmp_mod_dir)
|
||||
# Use go-mod-update-modules to set the full SRC_URI and LICENSE
|
||||
classes.append("go-mod-update-modules")
|
||||
extravalues["run_tasks"] = "update_modules"
|
||||
|
||||
stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
|
||||
go_mod = json.loads(stdout)
|
||||
go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
|
||||
with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir:
|
||||
env = dict(os.environ)
|
||||
env["PATH"] += f":{go_bindir}"
|
||||
env['GOMODCACHE'] = tmp_mod_dir
|
||||
|
||||
localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
|
||||
extravalues.setdefault('extrafiles', {})
|
||||
stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True)
|
||||
go_mod = json.loads(stdout)
|
||||
go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
|
||||
|
||||
# Write the ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
|
||||
self.__go_mod(go_mod, srctree, localfilesdir, extravalues, d)
|
||||
localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
|
||||
extravalues.setdefault('extrafiles', {})
|
||||
|
||||
# Do generic license handling
|
||||
handle_license_vars(srctree, lines_before, handled, extravalues, d)
|
||||
self.__rewrite_lic_vars(lines_before)
|
||||
# Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
|
||||
basename = "{pn}-licenses.inc"
|
||||
filename = os.path.join(localfilesdir, basename)
|
||||
with open(filename, "w") as f:
|
||||
f.write("# FROM RECIPETOOL\n")
|
||||
extravalues['extrafiles'][f"../{basename}"] = filename
|
||||
|
||||
self.__rewrite_src_uri(lines_before)
|
||||
basename = "{pn}-go-mods.inc"
|
||||
filename = os.path.join(localfilesdir, basename)
|
||||
with open(filename, "w") as f:
|
||||
f.write("# FROM RECIPETOOL\n")
|
||||
extravalues['extrafiles'][f"../{basename}"] = filename
|
||||
|
||||
lines_before.append('require ${BPN}-licenses.inc')
|
||||
lines_before.append('require ${BPN}-go-mods.inc')
|
||||
lines_before.append(f'GO_IMPORT = "{go_import}"')
|
||||
# Do generic license handling
|
||||
d = bb.data.createCopy(tinfoil.config_data)
|
||||
handle_license_vars(srctree, lines_before, handled, extravalues, d)
|
||||
self.__rewrite_lic_vars(lines_before)
|
||||
|
||||
self.__rewrite_src_uri(lines_before)
|
||||
|
||||
lines_before.append('require ${BPN}-licenses.inc')
|
||||
lines_before.append('require ${BPN}-go-mods.inc')
|
||||
lines_before.append(f'GO_IMPORT = "{go_import}"')
|
||||
|
||||
def __update_lines_before(self, updated, newlines, lines_before):
|
||||
if updated:
|
||||
|
|
@ -210,10 +132,8 @@ class GoRecipeHandler(RecipeHandler):
|
|||
return updated
|
||||
|
||||
def __rewrite_lic_vars(self, lines_before):
|
||||
|
||||
def varfunc(varname, origvalue, op, newlines):
|
||||
if varname == 'LICENSE':
|
||||
return ' & '.join((origvalue, '${GO_MOD_LICENSES}')), None, -1, True
|
||||
import urllib.parse
|
||||
if varname == 'LIC_FILES_CHKSUM':
|
||||
new_licenses = []
|
||||
licenses = origvalue.split('\\')
|
||||
|
|
@ -235,7 +155,7 @@ class GoRecipeHandler(RecipeHandler):
|
|||
return origvalue, None, 0, True
|
||||
|
||||
updated, newlines = bb.utils.edit_metadata(
|
||||
lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
|
||||
lines_before, ['LIC_FILES_CHKSUM'], varfunc)
|
||||
return self.__update_lines_before(updated, newlines, lines_before)
|
||||
|
||||
def __rewrite_src_uri(self, lines_before):
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user