mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
5eb10c53d8
Major module version is a part of name, but not necessary part of the actual URL (See https://go.dev/ref/mod#module-path). Nevertheless, name detection function can't handle that suffix, so get rid of it to determine component name. For replaced modules that name might be different that the actual module name defined in go.mod file. (From OE-Core rev: 0cccfa1041d48f0ae3a2dc89a129cf7884fc08f0) Signed-off-by: Vyacheslav Yurkov <uvv.mail@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
780 lines
30 KiB
Python
780 lines
30 KiB
Python
# Recipe creation tool - go support plugin
|
|
#
|
|
# The code is based on golang internals. See the afftected
|
|
# methods for further reference and information.
|
|
#
|
|
# Copyright (C) 2023 Weidmueller GmbH & Co KG
|
|
# Author: Lukas Funke <lukas.funke@weidmueller.com>
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
|
|
from collections import namedtuple
|
|
from enum import Enum
|
|
from html.parser import HTMLParser
|
|
from recipetool.create import RecipeHandler, handle_license_vars
|
|
from recipetool.create import guess_license, tidy_licenses, fixup_license
|
|
from recipetool.create import determine_from_url
|
|
from urllib.error import URLError
|
|
|
|
import bb.utils
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import shutil
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
GoImport = namedtuple('GoImport', 'root vcs url suffix')
|
|
logger = logging.getLogger('recipetool')
|
|
CodeRepo = namedtuple(
|
|
'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor')
|
|
|
|
tinfoil = None
|
|
|
|
# Regular expression to parse pseudo semantic version
|
|
# see https://go.dev/ref/mod#pseudo-versions
|
|
re_pseudo_semver = re.compile(
|
|
r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$")
|
|
# Regular expression to parse semantic version
|
|
re_semver = re.compile(
|
|
r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
|
|
|
|
|
|
def tinfoil_init(instance):
|
|
global tinfoil
|
|
tinfoil = instance
|
|
|
|
|
|
class GoRecipeHandler(RecipeHandler):
|
|
"""Class to handle the go recipe creation"""
|
|
|
|
@staticmethod
|
|
def __ensure_go():
|
|
"""Check if the 'go' command is available in the recipes"""
|
|
recipe = "go-native"
|
|
if not tinfoil.recipes_parsed:
|
|
tinfoil.parse_recipes()
|
|
try:
|
|
rd = tinfoil.parse_recipe(recipe)
|
|
except bb.providers.NoProvider:
|
|
bb.error(
|
|
"Nothing provides '%s' which is required for the build" % (recipe))
|
|
bb.note(
|
|
"You will likely need to add a layer that provides '%s'" % (recipe))
|
|
return None
|
|
|
|
bindir = rd.getVar('STAGING_BINDIR_NATIVE')
|
|
gopath = os.path.join(bindir, 'go')
|
|
|
|
if not os.path.exists(gopath):
|
|
tinfoil.build_targets(recipe, 'addto_recipe_sysroot')
|
|
|
|
if not os.path.exists(gopath):
|
|
logger.error(
|
|
'%s required to process specified source, but %s did not seem to populate it' % 'go', recipe)
|
|
return None
|
|
|
|
return bindir
|
|
|
|
def __resolve_repository_static(self, modulepath):
|
|
"""Resolve the repository in a static manner
|
|
|
|
The method is based on the go implementation of
|
|
`repoRootFromVCSPaths` in
|
|
https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
|
|
"""
|
|
|
|
url = urllib.parse.urlparse("https://" + modulepath)
|
|
req = urllib.request.Request(url.geturl())
|
|
|
|
try:
|
|
resp = urllib.request.urlopen(req)
|
|
# Some modulepath are just redirects to github (or some other vcs
|
|
# hoster). Therefore, we check if this modulepath redirects to
|
|
# somewhere else
|
|
if resp.geturl() != url.geturl():
|
|
bb.debug(1, "%s is redirectred to %s" %
|
|
(url.geturl(), resp.geturl()))
|
|
url = urllib.parse.urlparse(resp.geturl())
|
|
modulepath = url.netloc + url.path
|
|
|
|
except URLError as url_err:
|
|
# This is probably because the module path
|
|
# contains the subdir and major path. Thus,
|
|
# we ignore this error for now
|
|
logger.debug(
|
|
1, "Failed to fetch page from [%s]: %s" % (url, str(url_err)))
|
|
|
|
host, _, _ = modulepath.partition('/')
|
|
|
|
class vcs(Enum):
|
|
pathprefix = "pathprefix"
|
|
regexp = "regexp"
|
|
type = "type"
|
|
repo = "repo"
|
|
check = "check"
|
|
schemelessRepo = "schemelessRepo"
|
|
|
|
# GitHub
|
|
vcsGitHub = {}
|
|
vcsGitHub[vcs.pathprefix] = "github.com"
|
|
vcsGitHub[vcs.regexp] = re.compile(
|
|
r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
|
|
vcsGitHub[vcs.type] = "git"
|
|
vcsGitHub[vcs.repo] = "https://\\g<root>"
|
|
|
|
# Bitbucket
|
|
vcsBitbucket = {}
|
|
vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
|
|
vcsBitbucket[vcs.regexp] = re.compile(
|
|
r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
|
|
vcsBitbucket[vcs.type] = "git"
|
|
vcsBitbucket[vcs.repo] = "https://\\g<root>"
|
|
|
|
# IBM DevOps Services (JazzHub)
|
|
vcsIBMDevOps = {}
|
|
vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
|
|
vcsIBMDevOps[vcs.regexp] = re.compile(
|
|
r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
|
|
vcsIBMDevOps[vcs.type] = "git"
|
|
vcsIBMDevOps[vcs.repo] = "https://\\g<root>"
|
|
|
|
# Git at Apache
|
|
vcsApacheGit = {}
|
|
vcsApacheGit[vcs.pathprefix] = "git.apache.org"
|
|
vcsApacheGit[vcs.regexp] = re.compile(
|
|
r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
|
|
vcsApacheGit[vcs.type] = "git"
|
|
vcsApacheGit[vcs.repo] = "https://\\g<root>"
|
|
|
|
# Git at OpenStack
|
|
vcsOpenStackGit = {}
|
|
vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
|
|
vcsOpenStackGit[vcs.regexp] = re.compile(
|
|
r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
|
|
vcsOpenStackGit[vcs.type] = "git"
|
|
vcsOpenStackGit[vcs.repo] = "https://\\g<root>"
|
|
|
|
# chiselapp.com for fossil
|
|
vcsChiselapp = {}
|
|
vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
|
|
vcsChiselapp[vcs.regexp] = re.compile(
|
|
r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
|
|
vcsChiselapp[vcs.type] = "fossil"
|
|
vcsChiselapp[vcs.repo] = "https://\\g<root>"
|
|
|
|
# General syntax for any server.
|
|
# Must be last.
|
|
vcsGeneralServer = {}
|
|
vcsGeneralServer[vcs.regexp] = re.compile(
|
|
"(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$")
|
|
vcsGeneralServer[vcs.schemelessRepo] = True
|
|
|
|
vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps,
|
|
vcsApacheGit, vcsOpenStackGit, vcsChiselapp,
|
|
vcsGeneralServer]
|
|
|
|
if modulepath.startswith("example.net") or modulepath == "rsc.io":
|
|
logger.warning("Suspicious module path %s" % modulepath)
|
|
return None
|
|
if modulepath.startswith("http:") or modulepath.startswith("https:"):
|
|
logger.warning("Import path should not start with %s %s" %
|
|
("http", "https"))
|
|
return None
|
|
|
|
rootpath = None
|
|
vcstype = None
|
|
repourl = None
|
|
suffix = None
|
|
|
|
for srv in vcsPaths:
|
|
m = srv[vcs.regexp].match(modulepath)
|
|
if vcs.pathprefix in srv:
|
|
if host == srv[vcs.pathprefix]:
|
|
rootpath = m.group('root')
|
|
vcstype = srv[vcs.type]
|
|
repourl = m.expand(srv[vcs.repo])
|
|
suffix = m.group('suffix')
|
|
break
|
|
elif m and srv[vcs.schemelessRepo]:
|
|
rootpath = m.group('root')
|
|
vcstype = m[vcs.type]
|
|
repourl = m[vcs.repo]
|
|
suffix = m.group('suffix')
|
|
break
|
|
|
|
return GoImport(rootpath, vcstype, repourl, suffix)
|
|
|
|
def __resolve_repository_dynamic(self, modulepath):
|
|
"""Resolve the repository root in a dynamic manner.
|
|
|
|
The method is based on the go implementation of
|
|
`repoRootForImportDynamic` in
|
|
https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
|
|
"""
|
|
url = urllib.parse.urlparse("https://" + modulepath)
|
|
|
|
class GoImportHTMLParser(HTMLParser):
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.__srv = []
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == 'meta' and list(
|
|
filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
|
|
content = list(
|
|
filter(lambda a: (a[0] == 'content'), attrs))
|
|
if content:
|
|
self.__srv = content[0][1].split()
|
|
|
|
@property
|
|
def import_prefix(self):
|
|
return self.__srv[0] if len(self.__srv) else None
|
|
|
|
@property
|
|
def vcs(self):
|
|
return self.__srv[1] if len(self.__srv) else None
|
|
|
|
@property
|
|
def repourl(self):
|
|
return self.__srv[2] if len(self.__srv) else None
|
|
|
|
url = url.geturl() + "?go-get=1"
|
|
req = urllib.request.Request(url)
|
|
|
|
try:
|
|
resp = urllib.request.urlopen(req)
|
|
|
|
except URLError as url_err:
|
|
logger.warning(
|
|
"Failed to fetch page from [%s]: %s", url, str(url_err))
|
|
return None
|
|
|
|
parser = GoImportHTMLParser()
|
|
parser.feed(resp.read().decode('utf-8'))
|
|
parser.close()
|
|
|
|
return GoImport(parser.import_prefix, parser.vcs, parser.repourl, None)
|
|
|
|
def __resolve_from_golang_proxy(self, modulepath, version):
|
|
"""
|
|
Resolves repository data from golang proxy
|
|
"""
|
|
url = urllib.parse.urlparse("https://proxy.golang.org/"
|
|
+ modulepath
|
|
+ "/@v/"
|
|
+ version
|
|
+ ".info")
|
|
|
|
# Transform url to lower case, golang proxy doesn't like mixed case
|
|
req = urllib.request.Request(url.geturl().lower())
|
|
|
|
try:
|
|
resp = urllib.request.urlopen(req)
|
|
except URLError as url_err:
|
|
logger.warning(
|
|
"Failed to fetch page from [%s]: %s", url, str(url_err))
|
|
return None
|
|
|
|
golang_proxy_res = resp.read().decode('utf-8')
|
|
modinfo = json.loads(golang_proxy_res)
|
|
|
|
if modinfo and 'Origin' in modinfo:
|
|
origin = modinfo['Origin']
|
|
_root_url = urllib.parse.urlparse(origin['URL'])
|
|
|
|
# We normalize the repo URL since we don't want the scheme in it
|
|
_subdir = origin['Subdir'] if 'Subdir' in origin else None
|
|
_root, _, _ = self.__split_path_version(modulepath)
|
|
if _subdir:
|
|
_root = _root[:-len(_subdir)].strip('/')
|
|
|
|
_commit = origin['Hash']
|
|
_vcs = origin['VCS']
|
|
return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit)
|
|
|
|
return None
|
|
|
|
def __resolve_repository(self, modulepath):
|
|
"""
|
|
Resolves src uri from go module-path
|
|
"""
|
|
repodata = self.__resolve_repository_static(modulepath)
|
|
if not repodata or not repodata.url:
|
|
repodata = self.__resolve_repository_dynamic(modulepath)
|
|
if not repodata or not repodata.url:
|
|
logger.error(
|
|
"Could not resolve repository for module path '%s'" % modulepath)
|
|
# There is no way to recover from this
|
|
sys.exit(14)
|
|
if repodata:
|
|
logger.debug(1, "Resolved download path for import '%s' => %s" % (
|
|
modulepath, repodata.url))
|
|
return repodata
|
|
|
|
def __split_path_version(self, path):
|
|
i = len(path)
|
|
dot = False
|
|
for j in range(i, 0, -1):
|
|
if path[j - 1] < '0' or path[j - 1] > '9':
|
|
break
|
|
if path[j - 1] == '.':
|
|
dot = True
|
|
break
|
|
i = j - 1
|
|
|
|
if i <= 1 or i == len(
|
|
path) or path[i - 1] != 'v' or path[i - 2] != '/':
|
|
return path, "", True
|
|
|
|
prefix, pathMajor = path[:i - 2], path[i - 2:]
|
|
if dot or len(
|
|
pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1":
|
|
return path, "", False
|
|
|
|
return prefix, pathMajor, True
|
|
|
|
def __get_path_major(self, pathMajor):
|
|
if not pathMajor:
|
|
return ""
|
|
|
|
if pathMajor[0] != '/' and pathMajor[0] != '.':
|
|
logger.error(
|
|
"pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor)
|
|
|
|
if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"):
|
|
pathMajor = pathMajor[:len("-unstable") - 2]
|
|
|
|
return pathMajor[1:]
|
|
|
|
def __build_coderepo(self, repo, path):
|
|
codedir = ""
|
|
pathprefix, pathMajor, _ = self.__split_path_version(path)
|
|
if repo.root == path:
|
|
pathprefix = path
|
|
elif path.startswith(repo.root):
|
|
codedir = pathprefix[len(repo.root):].strip('/')
|
|
|
|
pseudoMajor = self.__get_path_major(pathMajor)
|
|
|
|
logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'",
|
|
repo.root, codedir, pathprefix, pathMajor, pseudoMajor)
|
|
|
|
return CodeRepo(path, repo.root, codedir,
|
|
pathMajor, pathprefix, pseudoMajor)
|
|
|
|
def __resolve_version(self, repo, path, version):
|
|
hash = None
|
|
coderoot = self.__build_coderepo(repo, path)
|
|
|
|
def vcs_fetch_all():
|
|
tmpdir = tempfile.mkdtemp()
|
|
clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir)
|
|
bb.process.run(clone_cmd)
|
|
log_cmd = "git log --all --pretty='%H %d' --decorate=short"
|
|
output, _ = bb.process.run(
|
|
log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
|
|
bb.utils.prunedir(tmpdir)
|
|
return output.strip().split('\n')
|
|
|
|
def vcs_fetch_remote(tag):
|
|
# add * to grab ^{}
|
|
refs = {}
|
|
ls_remote_cmd = "git ls-remote -q --tags {} {}*".format(
|
|
repo.url, tag)
|
|
output, _ = bb.process.run(ls_remote_cmd)
|
|
output = output.strip().split('\n')
|
|
for line in output:
|
|
f = line.split(maxsplit=1)
|
|
if len(f) != 2:
|
|
continue
|
|
|
|
for prefix in ["HEAD", "refs/heads/", "refs/tags/"]:
|
|
if f[1].startswith(prefix):
|
|
refs[f[1][len(prefix):]] = f[0]
|
|
|
|
for key, hash in refs.items():
|
|
if key.endswith(r"^{}"):
|
|
refs[key.strip(r"^{}")] = hash
|
|
|
|
return refs[tag]
|
|
|
|
m_pseudo_semver = re_pseudo_semver.match(version)
|
|
|
|
if m_pseudo_semver:
|
|
remote_refs = vcs_fetch_all()
|
|
short_commit = m_pseudo_semver.group('commithash')
|
|
for l in remote_refs:
|
|
r = l.split(maxsplit=1)
|
|
sha1 = r[0] if len(r) else None
|
|
if not sha1:
|
|
logger.error(
|
|
"Ups: could not resolve abbref commit for %s" % short_commit)
|
|
|
|
elif sha1.startswith(short_commit):
|
|
hash = sha1
|
|
break
|
|
else:
|
|
m_semver = re_semver.match(version)
|
|
if m_semver:
|
|
|
|
def get_sha1_remote(re):
|
|
rsha1 = None
|
|
for line in remote_refs:
|
|
# Split lines of the following format:
|
|
# 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag)
|
|
lineparts = line.split(maxsplit=1)
|
|
sha1 = lineparts[0] if len(lineparts) else None
|
|
refstring = lineparts[1] if len(
|
|
lineparts) == 2 else None
|
|
if refstring:
|
|
# Normalize tag string and split in case of multiple
|
|
# regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...)
|
|
refs = refstring.strip('(), ').split(',')
|
|
for ref in refs:
|
|
if re.match(ref.strip()):
|
|
rsha1 = sha1
|
|
return rsha1
|
|
|
|
semver = "v" + m_semver.group('major') + "."\
|
|
+ m_semver.group('minor') + "."\
|
|
+ m_semver.group('patch') \
|
|
+ (("-" + m_semver.group('prerelease'))
|
|
if m_semver.group('prerelease') else "")
|
|
|
|
tag = os.path.join(
|
|
coderoot.codeDir, semver) if coderoot.codeDir else semver
|
|
|
|
# probe tag using 'ls-remote', which is faster than fetching
|
|
# complete history
|
|
hash = vcs_fetch_remote(tag)
|
|
if not hash:
|
|
# backup: fetch complete history
|
|
remote_refs = vcs_fetch_all()
|
|
hash = get_sha1_remote(
|
|
re.compile(fr"(tag:|HEAD ->) ({tag})"))
|
|
|
|
logger.debug(
|
|
"Resolving commit for tag '%s' -> '%s'", tag, hash)
|
|
return hash
|
|
|
|
def __generate_srcuri_inline_fcn(self, path, version, replaces=None):
|
|
"""Generate SRC_URI functions for go imports"""
|
|
|
|
logger.info("Resolving repository for module %s", path)
|
|
# First try to resolve repo and commit from golang proxy
|
|
# Most info is already there and we don't have to go through the
|
|
# repository or even perform the version resolve magic
|
|
golang_proxy_info = self.__resolve_from_golang_proxy(path, version)
|
|
if golang_proxy_info:
|
|
repo = golang_proxy_info[0]
|
|
commit = golang_proxy_info[1]
|
|
else:
|
|
# Fallback
|
|
# Resolve repository by 'hand'
|
|
repo = self.__resolve_repository(path)
|
|
commit = self.__resolve_version(repo, path, version)
|
|
|
|
url = urllib.parse.urlparse(repo.url)
|
|
repo_url = url.netloc + url.path
|
|
|
|
coderoot = self.__build_coderepo(repo, path)
|
|
|
|
inline_fcn = "${@go_src_uri("
|
|
inline_fcn += f"'{repo_url}','{version}'"
|
|
if repo_url != path:
|
|
inline_fcn += f",path='{path}'"
|
|
if coderoot.codeDir:
|
|
inline_fcn += f",subdir='{coderoot.codeDir}'"
|
|
if repo.vcs != 'git':
|
|
inline_fcn += f",vcs='{repo.vcs}'"
|
|
if replaces:
|
|
inline_fcn += f",replaces='{replaces}'"
|
|
if coderoot.pathMajor:
|
|
inline_fcn += f",pathmajor='{coderoot.pathMajor}'"
|
|
inline_fcn += ")}"
|
|
|
|
return inline_fcn, commit
|
|
|
|
def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d):
|
|
|
|
import re
|
|
src_uris = []
|
|
src_revs = []
|
|
|
|
def generate_src_rev(path, version, commithash):
|
|
src_rev = f"# {path}@{version} => {commithash}\n"
|
|
# Ups...maybe someone manipulated the source repository and the
|
|
# version or commit could not be resolved. This is a sign of
|
|
# a) the supply chain was manipulated (bad)
|
|
# b) the implementation for the version resolving didn't work
|
|
# anymore (less bad)
|
|
if not commithash:
|
|
src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
|
src_rev += f"#!!! Could not resolve version !!!\n"
|
|
src_rev += f"#!!! Possible supply chain attack !!!\n"
|
|
src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
|
src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\""
|
|
|
|
return src_rev
|
|
|
|
# we first go over replacement list, because we are essentialy
|
|
# interested only in the replaced path
|
|
if go_mod['Replace']:
|
|
for replacement in go_mod['Replace']:
|
|
oldpath = replacement['Old']['Path']
|
|
path = replacement['New']['Path']
|
|
version = ''
|
|
if 'Version' in replacement['New']:
|
|
version = replacement['New']['Version']
|
|
|
|
if os.path.exists(os.path.join(srctree, path)):
|
|
# the module refers to the local path, remove it from requirement list
|
|
# because it's a local module
|
|
go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath]
|
|
else:
|
|
# Replace the path and the version, so we don't iterate replacement list anymore
|
|
for require in go_mod['Require']:
|
|
if require['Path'] == oldpath:
|
|
require.update({'Path': path, 'Version': version})
|
|
break
|
|
|
|
for require in go_mod['Require']:
|
|
path = require['Path']
|
|
version = require['Version']
|
|
|
|
inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
|
|
path, version)
|
|
src_uris.append(inline_fcn)
|
|
src_revs.append(generate_src_rev(path, version, commithash))
|
|
|
|
# strip version part from module URL /vXX
|
|
baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
|
|
pn, _ = determine_from_url(baseurl)
|
|
go_mods_basename = "%s-modules.inc" % pn
|
|
|
|
go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
|
|
with open(go_mods_filename, "w") as f:
|
|
# We introduce this indirection to make the tests a little easier
|
|
f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n")
|
|
f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n")
|
|
for uri in src_uris:
|
|
f.write(" " + uri + " \\\n")
|
|
f.write("\"\n\n")
|
|
for rev in src_revs:
|
|
f.write(rev + "\n")
|
|
|
|
extravalues['extrafiles'][go_mods_basename] = go_mods_filename
|
|
|
|
def __go_run_cmd(self, cmd, cwd, d):
|
|
return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')),
|
|
shell=True, cwd=cwd)
|
|
|
|
def __go_native_version(self, d):
|
|
stdout, _ = self.__go_run_cmd("go version", None, d)
|
|
m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout)
|
|
major = int(m.group(2))
|
|
minor = int(m.group(3))
|
|
patch = int(m.group(4))
|
|
|
|
return major, minor, patch
|
|
|
|
def __go_mod_patch(self, srctree, localfilesdir, extravalues, d):
|
|
|
|
patchfilename = "go.mod.patch"
|
|
go_native_version_major, go_native_version_minor, _ = self.__go_native_version(
|
|
d)
|
|
self.__go_run_cmd("go mod tidy -go=%d.%d" %
|
|
(go_native_version_major, go_native_version_minor), srctree, d)
|
|
stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
|
|
|
|
# Create patch in order to upgrade go version
|
|
self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d)
|
|
# Restore original state
|
|
self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d)
|
|
|
|
go_mod = json.loads(stdout)
|
|
tmpfile = os.path.join(localfilesdir, patchfilename)
|
|
shutil.move(os.path.join(srctree, patchfilename), tmpfile)
|
|
|
|
extravalues['extrafiles'][patchfilename] = tmpfile
|
|
|
|
return go_mod, patchfilename
|
|
|
|
def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d):
|
|
# Perform vendoring to retrieve the correct modules.txt
|
|
tmp_vendor_dir = tempfile.mkdtemp()
|
|
|
|
# -v causes to go to print modules.txt to stderr
|
|
_, stderr = self.__go_run_cmd(
|
|
"go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d)
|
|
|
|
modules_txt_basename = "modules.txt"
|
|
modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename)
|
|
with open(modules_txt_filename, "w") as f:
|
|
f.write(stderr)
|
|
|
|
extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename
|
|
|
|
licenses = []
|
|
lic_files_chksum = []
|
|
licvalues = guess_license(tmp_vendor_dir, d)
|
|
shutil.rmtree(tmp_vendor_dir)
|
|
|
|
if licvalues:
|
|
for licvalue in licvalues:
|
|
license = licvalue[0]
|
|
lics = tidy_licenses(fixup_license(license))
|
|
lics = [lic for lic in lics if lic not in licenses]
|
|
if len(lics):
|
|
licenses.extend(lics)
|
|
lic_files_chksum.append(
|
|
'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2]))
|
|
|
|
# strip version part from module URL /vXX
|
|
baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
|
|
pn, _ = determine_from_url(baseurl)
|
|
licenses_basename = "%s-licenses.inc" % pn
|
|
|
|
licenses_filename = os.path.join(localfilesdir, licenses_basename)
|
|
with open(licenses_filename, "w") as f:
|
|
f.write("GO_MOD_LICENSES = \"%s\"\n\n" %
|
|
' & '.join(sorted(licenses, key=str.casefold)))
|
|
# We introduce this indirection to make the tests a little easier
|
|
f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n")
|
|
f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n")
|
|
for lic in lic_files_chksum:
|
|
f.write(" " + lic + " \\\n")
|
|
f.write("\"\n")
|
|
|
|
extravalues['extrafiles'][licenses_basename] = licenses_filename
|
|
|
|
def process(self, srctree, classes, lines_before,
|
|
lines_after, handled, extravalues):
|
|
|
|
if 'buildsystem' in handled:
|
|
return False
|
|
|
|
files = RecipeHandler.checkfiles(srctree, ['go.mod'])
|
|
if not files:
|
|
return False
|
|
|
|
d = bb.data.createCopy(tinfoil.config_data)
|
|
go_bindir = self.__ensure_go()
|
|
if not go_bindir:
|
|
sys.exit(14)
|
|
|
|
d.prependVar('PATH', '%s:' % go_bindir)
|
|
handled.append('buildsystem')
|
|
classes.append("go-vendor")
|
|
|
|
stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
|
|
|
|
go_mod = json.loads(stdout)
|
|
go_import = go_mod['Module']['Path']
|
|
go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
|
|
go_version_major = int(go_version_match.group(1))
|
|
go_version_minor = int(go_version_match.group(2))
|
|
src_uris = []
|
|
|
|
localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
|
|
extravalues.setdefault('extrafiles', {})
|
|
|
|
# Use an explicit name determined from the module name because it
|
|
# might differ from the actual URL for replaced modules
|
|
# strip version part from module URL /vXX
|
|
baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
|
|
pn, _ = determine_from_url(baseurl)
|
|
|
|
# go.mod files with version < 1.17 may not include all indirect
|
|
# dependencies. Thus, we have to upgrade the go version.
|
|
if go_version_major == 1 and go_version_minor < 17:
|
|
logger.warning(
|
|
"go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
|
|
go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir,
|
|
extravalues, d)
|
|
src_uris.append(
|
|
"file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename))
|
|
|
|
# Check whether the module is vendored. If so, we have nothing to do.
|
|
# Otherwise we gather all dependencies and add them to the recipe
|
|
if not os.path.exists(os.path.join(srctree, "vendor")):
|
|
|
|
# Write additional $BPN-modules.inc file
|
|
self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d)
|
|
lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"")
|
|
lines_before.append("require %s-licenses.inc" % (pn))
|
|
|
|
self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
|
|
|
|
self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d)
|
|
lines_before.append("require %s-modules.inc" % (pn))
|
|
|
|
# Do generic license handling
|
|
handle_license_vars(srctree, lines_before, handled, extravalues, d)
|
|
self.__rewrite_lic_uri(lines_before)
|
|
|
|
lines_before.append("GO_IMPORT = \"{}\"".format(baseurl))
|
|
lines_before.append("SRCREV_FORMAT = \"${BPN}\"")
|
|
|
|
def __update_lines_before(self, updated, newlines, lines_before):
|
|
if updated:
|
|
del lines_before[:]
|
|
for line in newlines:
|
|
# Hack to avoid newlines that edit_metadata inserts
|
|
if line.endswith('\n'):
|
|
line = line[:-1]
|
|
lines_before.append(line)
|
|
return updated
|
|
|
|
def __rewrite_lic_uri(self, lines_before):
|
|
|
|
def varfunc(varname, origvalue, op, newlines):
|
|
if varname == 'LIC_FILES_CHKSUM':
|
|
new_licenses = []
|
|
licenses = origvalue.split('\\')
|
|
for license in licenses:
|
|
if not license:
|
|
logger.warning("No license file was detected for the main module!")
|
|
# the license list of the main recipe must be empty
|
|
# this can happen for example in case of CLOSED license
|
|
# Fall through to complete recipe generation
|
|
continue
|
|
license = license.strip()
|
|
uri, chksum = license.split(';', 1)
|
|
url = urllib.parse.urlparse(uri)
|
|
new_uri = os.path.join(
|
|
url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
|
|
new_licenses.append(new_uri)
|
|
|
|
return new_licenses, None, -1, True
|
|
return origvalue, None, 0, True
|
|
|
|
updated, newlines = bb.utils.edit_metadata(
|
|
lines_before, ['LIC_FILES_CHKSUM'], varfunc)
|
|
return self.__update_lines_before(updated, newlines, lines_before)
|
|
|
|
def __rewrite_src_uri(self, lines_before, additional_uris = []):
|
|
|
|
def varfunc(varname, origvalue, op, newlines):
|
|
if varname == 'SRC_URI':
|
|
src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"]
|
|
src_uri.extend(additional_uris)
|
|
return src_uri, None, -1, True
|
|
return origvalue, None, 0, True
|
|
|
|
updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
|
|
return self.__update_lines_before(updated, newlines, lines_before)
|
|
|
|
|
|
def register_recipe_handlers(handlers):
|
|
handlers.append((GoRecipeHandler(), 60))
|