mirror of
git://git.yoctoproject.org/meta-virtualization.git
synced 2025-07-05 05:15:25 +02:00
881 lines
42 KiB
Python
Executable File
881 lines
42 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
# go-dep processor
|
|
#
|
|
# Copyright (C) 2022 Bruce Ashfield
|
|
# Copyright (C) 2023 Chen Qi
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
import argparse
|
|
from collections import OrderedDict
|
|
import subprocess
|
|
import textwrap
|
|
import re
|
|
|
|
# This switch is used to make this script error out ASAP, mainly for debugging purpose
|
|
ERROR_OUT_ON_FETCH_AND_CHECKOUT_FAILURE = False
|
|
|
|
logger = logging.getLogger('oe-go-mod-autogen')
|
|
loggerhandler = logging.StreamHandler()
|
|
loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
|
|
logger.addHandler(loggerhandler)
|
|
logger.setLevel(logging.INFO)
|
|
|
|
class GoModTool(object):
|
|
def __init__(self, repo, rev, workdir):
|
|
self.repo = repo
|
|
self.rev = rev
|
|
self.workdir = workdir
|
|
|
|
# Stores the actual module name and its related information
|
|
# {module: (repo_url, repo_dest_dir, fullsrcrev)}
|
|
self.modules_repoinfo = {}
|
|
|
|
# {module_name: (url, version, destdir, fullsrcrev)}
|
|
#
|
|
# url: place to get the source codes, we only support git repo
|
|
# version: module version, git tag or git rev
|
|
# destdir: place to put the fetched source codes
|
|
# fullsrcrev: full src rev which is the value of SRC_REV
|
|
#
|
|
# e.g.
|
|
# For 'github.com/Masterminds/semver/v3 v3.1.1' in go.mod:
|
|
# module_name = github.com/Masterminds/semver/v3
|
|
# url = https://github.com/Masterminds/semver
|
|
# version = v3.1.1
|
|
# destdir = ${WORKDIR}/${BP}/src/${GO_IMPORT}/vendor/github.com/Masterminds/semver/v3
|
|
# fullsrcrev = d387ce7889a157b19ad7694dba39a562051f41b0
|
|
self.modules_require = OrderedDict()
|
|
|
|
# {orig_module: (actual_module, actual_version)}
|
|
self.modules_replace = OrderedDict()
|
|
|
|
# Unhandled modules
|
|
self.modules_unhandled = OrderedDict()
|
|
|
|
# store subpaths used to form srcpath
|
|
# {actual_module_name: subpath}
|
|
self.modules_subpaths = OrderedDict()
|
|
|
|
# modules's actual source paths, record those that are not the same with the module itself
|
|
self.modules_srcpaths = OrderedDict()
|
|
|
|
# store lines, comment removed
|
|
self.require_lines = []
|
|
self.replace_lines = []
|
|
|
|
# fetch repo
|
|
self.fetch_and_checkout_repo(self.repo.split('://')[1], self.repo, self.rev, checkout=True, get_subpath=False)
|
|
|
|
def show_go_mod_info(self):
|
|
# Print modules_require, modules_replace and modules_unhandled
|
|
print("modules required:")
|
|
for m in self.modules_require:
|
|
url, version, destdir, fullrev = self.modules_require[m]
|
|
print("%s %s %s %s" % (m, version, url, fullrev))
|
|
|
|
print("modules replace:")
|
|
for m in self.modules_replace:
|
|
actual_module, actual_version = self.modules_replace[m]
|
|
print("%s => %s %s" % (m, actual_module, actual_version))
|
|
|
|
print("modules unhandled:")
|
|
for m in self.modules_unhandled:
|
|
reason = self.modules_unhandled[m]
|
|
print("%s unhandled: %s" % (m, reason))
|
|
|
|
def parse(self):
|
|
# check if this repo needs autogen
|
|
repo_url, repo_dest_dir, repo_fullrev = self.modules_repoinfo[self.repo.split('://')[1]]
|
|
if os.path.isdir(os.path.join(repo_dest_dir, 'vendor')):
|
|
logger.info("vendor directory already exists for %s, no need to add other repos" % self.repo)
|
|
return
|
|
go_mod_file = os.path.join(repo_dest_dir, 'go.mod')
|
|
if not os.path.exists(go_mod_file):
|
|
logger.info("go.mod file does not exist for %s, no need to add other repos" % self.repo)
|
|
return
|
|
self.parse_go_mod(go_mod_file)
|
|
self.show_go_mod_info()
|
|
|
|
def fetch_and_checkout_repo(self, module_name, repo_url, rev, default_protocol='https://', checkout=False, get_subpath=True):
|
|
"""
|
|
Fetch repo_url to <workdir>/repos/repo_base_name
|
|
"""
|
|
protocol = default_protocol
|
|
if '://' in repo_url:
|
|
repo_url_final = repo_url
|
|
else:
|
|
repo_url_final = default_protocol + repo_url
|
|
logger.debug("fetch and checkout %s %s" % (repo_url_final, rev))
|
|
repos_dir = os.path.join(self.workdir, 'repos')
|
|
if not os.path.exists(repos_dir):
|
|
os.makedirs(repos_dir)
|
|
repo_basename = repo_url.split('/')[-1].split('.git')[0]
|
|
repo_dest_dir = os.path.join(repos_dir, repo_basename)
|
|
module_last_name = module_name.split('/')[-1]
|
|
git_action = "fetch"
|
|
if os.path.exists(repo_dest_dir):
|
|
if checkout:
|
|
# check if current HEAD is rev
|
|
try:
|
|
headrev = subprocess.check_output('git rev-list -1 HEAD', shell=True, cwd=repo_dest_dir).decode('utf-8').strip()
|
|
requiredrev = subprocess.check_output('git rev-list -1 %s 2>/dev/null || git rev-list -1 %s/%s' % (rev, module_last_name, rev), shell=True, cwd=repo_dest_dir).decode('utf-8').strip()
|
|
if headrev == requiredrev:
|
|
logger.info("%s has already been fetched and checked out as required, skipping" % repo_url)
|
|
self.modules_repoinfo[module_name] = (repo_url, repo_dest_dir, requiredrev)
|
|
return
|
|
else:
|
|
logger.info("HEAD of %s is not %s, will do a clean clone" % (repo_dest_dir, requiredrev))
|
|
git_action = "clone"
|
|
except:
|
|
logger.info("'git rev-list' in %s failed, will do a clean clone" % repo_dest_dir)
|
|
git_action = "clone"
|
|
else:
|
|
# determine if the current repo points to the desired remote repo
|
|
try:
|
|
remote_origin_url = subprocess.check_output('git config --get remote.origin.url', shell=True, cwd=repo_dest_dir).decode('utf-8').strip()
|
|
if remote_origin_url.endswith('.git'):
|
|
if not repo_url_final.endswith('.git'):
|
|
remote_origin_url = remote_origin_url[:-4]
|
|
else:
|
|
if repo_url_final.endswith('.git'):
|
|
remote_origin_url = remote_origin_url + '.git'
|
|
if remote_origin_url != repo_url_final:
|
|
logger.info("remote.origin.url for %s is not %s, will do a clean clone" % (repo_dest_dir, repo_url_final))
|
|
git_action = "clone"
|
|
except:
|
|
logger.info("'git config --get remote.origin.url' in %s failed, will do a clean clone" % repo_dest_dir)
|
|
git_action = "clone"
|
|
else:
|
|
# No local repo, clone it.
|
|
git_action = "clone"
|
|
|
|
if git_action == "clone":
|
|
logger.info("Removing %s" % repo_dest_dir)
|
|
subprocess.check_call('rm -rf %s' % repo_dest_dir, shell=True)
|
|
|
|
# clone/fetch repo
|
|
try:
|
|
git_cwd = repos_dir if git_action == "clone" else repo_dest_dir
|
|
logger.info("git %s %s in %s" % (git_action, repo_url_final, git_cwd))
|
|
subprocess.check_call('git %s %s >/dev/null 2>&1' % (git_action, repo_url_final), shell=True, cwd=git_cwd)
|
|
except:
|
|
logger.warning("Failed to %s %s in %s" % (git_action, repo_url_final, git_cwd))
|
|
return
|
|
|
|
def get_requiredrev(get_subpath):
|
|
import re
|
|
# check if rev is a revision or a version
|
|
if len(rev) == 12 and re.match('[0-9a-f]+', rev):
|
|
rev_is_version = False
|
|
else:
|
|
rev_is_version = True
|
|
|
|
# if rev is not a version, 'git rev-list -1 <rev>' should just succeed!
|
|
if not rev_is_version:
|
|
try:
|
|
rev_return = subprocess.check_output('git rev-list -1 %s 2>/dev/null' % rev, shell=True, cwd=repo_dest_dir).decode('utf-8').strip()
|
|
if get_subpath:
|
|
cmd = 'git branch -M toremove && git checkout -b check_subpath %s && git branch -D toremove' % rev_return
|
|
subprocess.check_call(cmd, shell=True, cwd=repo_dest_dir)
|
|
# try to get the subpath for this module
|
|
module_name_parts = module_name.split('/')
|
|
while (len(module_name_parts) > 0):
|
|
subpath = '/'.join(module_name_parts)
|
|
dir_to_check = repo_dest_dir + '/' + '/'.join(module_name_parts)
|
|
if os.path.isdir(dir_to_check):
|
|
self.modules_subpaths[module_name] = subpath
|
|
break
|
|
else:
|
|
module_name_parts.pop(0)
|
|
return rev_return
|
|
except:
|
|
logger.warning("Revision (%s) not in repo(%s)" % (rev, repo_dest_dir))
|
|
return None
|
|
|
|
# the following codes deals with case where rev is a version
|
|
# determine the longest match tag, in this way, we can get the current srcpath to be used in relocation.inc
|
|
# we first get the initial tag, which is formed from module_name and rev
|
|
module_parts = module_name.split('/')
|
|
if rev.startswith(module_parts[-1] + '.'):
|
|
tag = '/'.join(module_parts[:-1]) + '/' + rev
|
|
last_module_part_replaced = True
|
|
else:
|
|
tag = '/'.join(module_parts) + '/' + rev
|
|
last_module_part_replaced = False
|
|
logger.debug("use %s as the initial tag for %s" % (tag, module_name))
|
|
tag_parts = tag.split('/')
|
|
while(len(tag_parts) > 0):
|
|
try:
|
|
rev_return = subprocess.check_output('git rev-list -1 %s 2>/dev/null' % tag, shell=True, cwd=repo_dest_dir).decode('utf-8').strip()
|
|
if len(tag_parts) > 1:
|
|
# ensure that the subpath exists
|
|
if get_subpath:
|
|
cmd = 'git branch -M toremove && git checkout -b check_subpath %s && git branch -D toremove' % rev_return
|
|
subprocess.check_call(cmd, shell=True, cwd=repo_dest_dir)
|
|
# get subpath for the actual_module_name
|
|
if last_module_part_replaced:
|
|
subpath = '/'.join(tag_parts[:-1]) + '/' + module_parts[-1]
|
|
if not os.path.isdir(repo_dest_dir + '/' + subpath):
|
|
subpath = '/'.join(tag_parts[:-1])
|
|
else:
|
|
subpath = '/'.join(tag_parts[:-1])
|
|
if not os.path.isdir(repo_dest_dir + '/' + subpath):
|
|
logger.warning("subpath (%s) derived from tag matching does not exist in %s" % (subpath, repo_dest_dir))
|
|
return None
|
|
self.modules_subpaths[module_name] = subpath
|
|
logger.info("modules_subpath[%s] = %s" % (module_name, subpath))
|
|
return rev_return
|
|
except:
|
|
tag_parts.pop(0)
|
|
tag = '/'.join(tag_parts)
|
|
logger.warning("No tag matching %s" % rev)
|
|
return None
|
|
|
|
requiredrev = get_requiredrev(get_subpath)
|
|
if requiredrev:
|
|
logger.info("Got module(%s) requiredrev: %s" % (module_name, requiredrev))
|
|
if checkout:
|
|
subprocess.check_call('git checkout -b gomodautogen %s' % requiredrev, shell=True, cwd=repo_dest_dir)
|
|
self.modules_repoinfo[module_name] = (repo_url, repo_dest_dir, requiredrev)
|
|
else:
|
|
logger.warning("Failed to get requiredrev, repo_url = %s, rev = %s, module_name = %s" % (repo_url, rev, module_name))
|
|
return None
|
|
|
|
def parse_go_mod(self, go_mod_path):
|
|
"""
|
|
Parse go.mod file to get the modules info
|
|
"""
|
|
# First we get the require and replace lines
|
|
# The parsing logic assumes the replace lines come *after* the require lines
|
|
inrequire = False
|
|
inreplace = False
|
|
with open(go_mod_path, 'r') as f:
|
|
lines = f.readlines()
|
|
for line in lines:
|
|
if line.startswith('require ('):
|
|
inrequire = True
|
|
continue
|
|
if line.startswith(')'):
|
|
inrequire = False
|
|
continue
|
|
if line.startswith('require ') or inrequire:
|
|
# we have one line require
|
|
require_line = line.lstrip('require ').split('//')[0].strip()
|
|
if require_line:
|
|
self.require_lines.append(require_line)
|
|
continue
|
|
# we can deal with requires and replaces separately because go.mod always writes requires before replaces
|
|
if line.startswith('replace ('):
|
|
inreplace = True
|
|
continue
|
|
if line.startswith(')'):
|
|
inreplace = False
|
|
continue
|
|
if line.startswith('replace ') or inreplace:
|
|
replace_line = line.lstrip('replace ').split('//')[0].strip()
|
|
if replace_line:
|
|
self.replace_lines.append(replace_line)
|
|
continue
|
|
#
|
|
# parse the require_lines and replace_lines to form self.modules_require and self.modules_replace
|
|
#
|
|
logger.debug("Parsing require_lines and replace_lines ...")
|
|
# A typical replace line is as below:
|
|
# github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
|
|
# It means that the github.com/hashicorp/golang-lru module is replaced by github.com/ktock/golang-lru
|
|
# with the version 'v0.5.5-0.20211029085301-ec551be6f75c'.
|
|
# So the destdir is vendor/github.com/hashicorp/golang-lru while the contents are from github.com/ktock/golang-lru
|
|
for line in self.replace_lines:
|
|
try:
|
|
orig_module, actual = line.split('=>')
|
|
print( f"replace line: orig: {orig_module} actual_version: {actual}")
|
|
actual_module, actual_version = actual.split()
|
|
print( f"replace line: actual: {actual_module} actual_version: {actual_version}")
|
|
orig_module = orig_module.strip()
|
|
actual_module = actual_module.strip()
|
|
actual_version = actual_version.strip()
|
|
self.modules_replace[orig_module] = (actual_module, actual_version)
|
|
except Exception as e:
|
|
print( f"exception {e} caught while parsing, ignoring line: {line}")
|
|
# sys.exit(1)
|
|
continue
|
|
#
|
|
# Typical require lines are as below:
|
|
# github.com/Masterminds/semver/v3 v3.1.1
|
|
# golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
|
|
#
|
|
# We need to first try https://<module_name>?=go-get=1 to see it contains
|
|
# line starting with '<meta name="go-import" content='.
|
|
#
|
|
# If so, get root-path vcs repo-url from content. See https://go.dev/ref/mod#vcs-find
|
|
# For example, the above 'wget https://golang.org/x/crypto?go-get=1' gives you
|
|
# <meta name="go-import" content="golang.org/x/crypto git https://go.googlesource.com/crypto">
|
|
# In such case, the self.modules_require has the following contents:
|
|
# module_name: golang.org/x/crypto
|
|
# url: https://go.googlesource.com/crypto
|
|
# version: v0.0.0-20220321153916-2c7772ba3064
|
|
# destdir: ${WORKDIR}/${BP}/src/import/vendor.fetch/golang.org/x/crypto
|
|
# fullsrcrev: 2c7772ba30643b7a2026cbea938420dce7c6384d (git rev-list -1 2c7772ba3064)
|
|
#
|
|
# If not, try https://pkg.go.dev/<module_name>, and find the 'Repository'.
|
|
# For example, 'wget https://pkg.go.dev/github.com/Masterminds/semver/v3' gives:
|
|
# github.com/Masterminds/semver
|
|
# In such case, the self.modules has the following contents:
|
|
# module_name: github.com/Masterminds/semver/v3
|
|
# url: https://github.com/Masterminds/semver
|
|
# version: v3.1.1
|
|
# destdir: ${WORKDIR}/${BP}/src/import/vendor.fetch/github.com/Masterminds/semver/v3
|
|
# fullsrcrev: 7bb0c843b53d6ad21a3f619cb22c4b442bb3ef3e (git rev-list -1 v3.1.1)
|
|
#
|
|
# Next, if the last component of <module_name> matches 'v[0-9]+',
|
|
# remove the last component and try wget https://<module_name_with_last_component_removed>?go-get=1,
|
|
# then try using the above matching method.
|
|
#
|
|
# Finally, we have a mapping of known modules to source trees that can
|
|
# be used to translate the go.mod entry to a repository. Currently this is
|
|
# part of the script, but could be read from .map files in the future.
|
|
#
|
|
for line in self.require_lines:
|
|
module_name, version = line.strip().split()
|
|
logger.debug("require line: %s" % line)
|
|
logger.debug("module_name = %s; version = %s" % (module_name, version))
|
|
# take the modules_replace into consideration to get the actual version and actual module name
|
|
# note that the module_name is used in destdir, and the actual_module_name and actual_version
|
|
# are used to determine the url and fullsrcrev
|
|
destdir = '${WORKDIR}/${BP}/src/import/vendor.fetch/%s' % module_name
|
|
actual_module_name = module_name
|
|
actual_version = version
|
|
if module_name in self.modules_replace:
|
|
actual_module_name, actual_version = self.modules_replace[module_name]
|
|
logger.debug("actual_module_name = %s; actual_version = %s" % (actual_module_name, actual_version))
|
|
url, fullsrcrev = self.get_url_srcrev(actual_module_name, actual_version)
|
|
logger.debug("url = %s; fullsrcrev = %s" % (url, fullsrcrev))
|
|
if url and fullsrcrev:
|
|
self.modules_require[module_name] = (url, version, destdir, fullsrcrev)
|
|
# form srcpath, actual_module_name/<subpath>
|
|
if actual_module_name in self.modules_subpaths:
|
|
subpath = self.modules_subpaths[actual_module_name]
|
|
srcpath = '%s/%s' % (actual_module_name, subpath)
|
|
self.modules_srcpaths[module_name] = srcpath
|
|
logger.info("self.modules_srcpaths[%s] = %s" % (module_name, srcpath))
|
|
else:
|
|
self.modules_srcpaths[module_name] = actual_module_name
|
|
else:
|
|
logger.warning("get_url_srcrev(%s, %s) failed" % (actual_module_name, actual_version))
|
|
if ERROR_OUT_ON_FETCH_AND_CHECKOUT_FAILURE:
|
|
sys.exit(1)
|
|
|
|
def use_wget_to_get_repo_url(self, wget_content_file, url_cache_file, module_name):
|
|
"""
|
|
Use wget to get repo_url for module_name, return None if not found
|
|
"""
|
|
try:
|
|
logger.info("wget -O %s https://%s?=go-get=1" % (wget_content_file, module_name))
|
|
subprocess.check_call('wget -O %s https://%s?=go-get=1' % (wget_content_file, module_name), shell=True)
|
|
with open(wget_content_file, 'r') as f:
|
|
for line in f.readlines():
|
|
if '<meta name="go-import" content=' in line:
|
|
logger.info("Succeed to find go-import content for %s" % module_name)
|
|
logger.debug("The line is %s" % line)
|
|
root_path, vcs, repo_url = line.split('content=')[1].split('"')[1].split()
|
|
logger.info("%s: %s %s %s" % (module_name, root_path, vcs, repo_url))
|
|
if vcs != 'git':
|
|
logger.warning('%s unhandled as its vcs is %s which is not supported by this script.' % (module_name, vcs))
|
|
unhandled_reason = 'vcs %s is not supported by this script' % vcs
|
|
self.modules_unhandled[module_name] = unhandled_reason
|
|
return None
|
|
with open(url_cache_file, 'w') as f:
|
|
f.write(repo_url)
|
|
return repo_url
|
|
except:
|
|
logger.info("wget -O %s https://%s?=go-get=1 failed" % (wget_content_file, module_name))
|
|
# if we cannot find repo url from https://<module_name>?=go-get=1, try https://pkg.go/dev/<module_name>
|
|
try:
|
|
logger.info("wget -O %s https://pkg.go.dev/%s" % (wget_content_file, module_name))
|
|
subprocess.check_call("wget -O %s https://pkg.go.dev/%s" % (wget_content_file, module_name), shell=True)
|
|
repo_url_found = False
|
|
with open(wget_content_file, 'r') as f:
|
|
in_repo_section = False
|
|
for line in f.readlines():
|
|
if '>Repository<' in line:
|
|
in_repo_section = True
|
|
continue
|
|
if in_repo_section:
|
|
newline = line.strip()
|
|
if newline != '' and not newline.startswith('<'):
|
|
repo_url = newline
|
|
repo_url_found = True
|
|
if "Repository URL not available" in repo_url:
|
|
repo_url_found = False
|
|
repo_url = ""
|
|
|
|
break
|
|
if repo_url_found:
|
|
logger.info("repo url for %s: %s" % (module_name, repo_url))
|
|
with open(url_cache_file, 'w') as f:
|
|
f.write(repo_url)
|
|
return repo_url
|
|
else:
|
|
unhandled_reason = 'cannot determine repo_url for %s' % module_name
|
|
self.modules_unhandled[module_name] = unhandled_reason
|
|
# This used to return, but we have the mapping step below to try
|
|
# as a final resort, leaving this here in case compatiblity issues
|
|
# arrive later due to the continued processing.
|
|
# return None
|
|
except:
|
|
logger.info("wget -O %s https://pkg.go.dev/%s failed" % (wget_content_file, module_name))
|
|
|
|
# Do we recognize this twice failed lookup ?
|
|
site_mapper = { "inet.af" : { "match" : re.compile(""),
|
|
"replace" : ""
|
|
}
|
|
}
|
|
|
|
# module name: inet.af/tcpproxy
|
|
# replacement: https://github.com/inetaf/tcpproxy
|
|
site_mapper["inet.af"]["match"] = re.compile(r"(inet\.af)/(.*)")
|
|
site_mapper["inet.af"]["replace"] = "https://github.com/inetaf/\\g<2>"
|
|
|
|
host, _, _ = module_name.partition('/')
|
|
|
|
## on failure, we could consider instructing the user to write their
|
|
## own url into the repo_url_cache file
|
|
##
|
|
## or we could look for a .repo_mapping file, and read/use it to do
|
|
## the mapping and carry that around per-project.
|
|
logger.info( "trying mapper lookup for %s (host: %s)" % (module_name,host))
|
|
|
|
try:
|
|
mapper = site_mapper[host]
|
|
m = mapper["match"].match(module_name)
|
|
repo_url = m.expand( mapper["replace"] )
|
|
|
|
logger.info( "mapper match for %s, returning %s" % (module_name,repo_url) )
|
|
#print( "new site: %s" % repo_url )
|
|
|
|
# clear any potentially staged reasons for failures above
|
|
self.modules_unhandled[module_name] = ""
|
|
|
|
with open(url_cache_file, 'w') as f:
|
|
f.write(repo_url)
|
|
return repo_url
|
|
except Exception as e:
|
|
unhandled_reason = 'cannot determine mapped repo_url for %s' % module_name
|
|
### XXXX: TODO. if there are more parts to be popped, we shouldn't give up
|
|
### on th emodule
|
|
####
|
|
#### and/ or check if there was already an entry from above, since that means
|
|
#### there was a more critcal error during the check and we should just
|
|
#### propagate the unhandled to the caller
|
|
####
|
|
self.modules_unhandled[module_name] = unhandled_reason
|
|
del self.modules_unhandled[module_name]
|
|
logger.info( "no mapper match, returning none: %s" % e )
|
|
return None
|
|
|
|
return None
|
|
|
|
def get_repo_url_rev(self, module_name, version):
|
|
"""
|
|
Return (repo_url, rev)
|
|
"""
|
|
import re
|
|
# First get rev from version
|
|
v = version.split('+incompatible')[0]
|
|
version_components = v.split('-')
|
|
if len(version_components) == 1:
|
|
rev = v
|
|
elif len(version_components) == 3:
|
|
if len(version_components[2]) == 12:
|
|
rev = version_components[2]
|
|
else:
|
|
rev = v
|
|
else:
|
|
rev = v
|
|
|
|
#
|
|
# Get repo_url
|
|
# We put a cache mechanism here, <wget_content_file>.repo_url.cache is used to store the repo url fetch before
|
|
#
|
|
wget_dir = os.path.join(self.workdir, 'wget-contents')
|
|
if not os.path.exists(wget_dir):
|
|
os.makedirs(wget_dir)
|
|
wget_content_file = os.path.join(wget_dir, module_name.replace('/', '_'))
|
|
url_cache_file = "%s.repo_url.cache" % wget_content_file
|
|
if os.path.exists(url_cache_file):
|
|
with open(url_cache_file, 'r') as f:
|
|
repo_url = f.readline().strip()
|
|
return (repo_url, rev)
|
|
module_name_parts = module_name.split('/')
|
|
while (len(module_name_parts) > 0):
|
|
module_name_to_check = '/'.join(module_name_parts)
|
|
logger.info("module_name_to_check: %s" % module_name_to_check)
|
|
repo_url = self.use_wget_to_get_repo_url(wget_content_file, url_cache_file, module_name_to_check)
|
|
if repo_url:
|
|
return (repo_url, rev)
|
|
else:
|
|
if module_name in self.modules_unhandled:
|
|
return (None, rev)
|
|
else:
|
|
module_name_parts.pop(-1)
|
|
|
|
unhandled_reason = 'cannot determine the repo for %s' % module_name
|
|
self.modules_unhandled[module_name] = unhandled_reason
|
|
return (None, rev)
|
|
|
|
def get_url_srcrev(self, module_name, version):
|
|
"""
|
|
Return url and fullsrcrev according to module_name and version
|
|
"""
|
|
repo_url, rev = self.get_repo_url_rev(module_name, version)
|
|
if not repo_url or not rev:
|
|
return (None, None)
|
|
self.fetch_and_checkout_repo(module_name, repo_url, rev)
|
|
if module_name in self.modules_repoinfo:
|
|
repo_url, repo_dest_dir, repo_fullrev = self.modules_repoinfo[module_name]
|
|
# remove the .git suffix to sync repos across modules with different versions and across recipes
|
|
if repo_url.endswith('.git'):
|
|
repo_url = repo_url[:-len('.git')]
|
|
return (repo_url, repo_fullrev)
|
|
else:
|
|
unhandled_reason = 'fetch_and_checkout_repo(%s, %s, %s) failed' % (module_name, repo_url, rev)
|
|
self.modules_unhandled[module_name] = unhandled_reason
|
|
return (None, None)
|
|
|
|
def gen_src_uri_inc(self):
|
|
"""
|
|
Generate src_uri.inc file containing SRC_URIs
|
|
"""
|
|
src_uri_inc_file = os.path.join(self.workdir, 'src_uri.inc')
|
|
# record the <name> after writting SRCREV_<name>, this is to avoid modules having the same basename resulting in same SRCREV_xxx
|
|
srcrev_name_recorded = []
|
|
# pre styhead releases
|
|
# SRC_URI += "git://%s;name=%s;protocol=https;nobranch=1;destsuffix=${WORKDIR}/${BP}/src/import/vendor.fetch/%s"
|
|
template = """# %s %s
|
|
# [1] git ls-remote %s %s
|
|
SRCREV_%s="%s"
|
|
SRC_URI += "git://%s;name=%s;protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}/vendor.fetch/%s"
|
|
|
|
"""
|
|
# We can't simply write SRC_URIs one by one in the order that go.mod specify them.
|
|
# Because the latter one might clean things up for the former one if the former one is a subpath of the latter one.
|
|
def take_first_len(elem):
|
|
return len(elem[0])
|
|
|
|
src_uri_contents = []
|
|
with open(src_uri_inc_file, 'w') as f:
|
|
for module in self.modules_require:
|
|
# {module_name: (url, version, destdir, fullsrcrev)}
|
|
repo_url, version, destdir, fullrev = self.modules_require[module]
|
|
if module in self.modules_replace:
|
|
actual_module_name, actual_version = self.modules_replace[module]
|
|
else:
|
|
actual_module_name, actual_version = (module, version)
|
|
if '://' in repo_url:
|
|
repo_url_noprotocol = repo_url.split('://')[1]
|
|
else:
|
|
repo_url_noprotocol = repo_url
|
|
if not repo_url.startswith('https://'):
|
|
repo_url = 'https://' + repo_url
|
|
name = module.split('/')[-1]
|
|
if name in srcrev_name_recorded:
|
|
name = '-'.join(module.split('/')[-2:])
|
|
src_uri_contents.append((actual_module_name, actual_version, repo_url, fullrev, name, fullrev, repo_url_noprotocol, name, actual_module_name))
|
|
srcrev_name_recorded.append(name)
|
|
# sort the src_uri_contents and then write it
|
|
src_uri_contents.sort(key=take_first_len)
|
|
for content in src_uri_contents:
|
|
try:
|
|
f.write(template % content)
|
|
except Exception as e:
|
|
logger.warning( "exception while writing src_uri.inc: %s" % e )
|
|
logger.info("%s generated" % src_uri_inc_file)
|
|
|
|
def gen_relocation_inc(self):
|
|
"""
|
|
Generate relocation.inc file
|
|
"""
|
|
relocation_inc_file = os.path.join(self.workdir, 'relocation.inc')
|
|
template = """export sites="%s"
|
|
|
|
do_compile:prepend() {
|
|
cd ${S}/src/import
|
|
for s in $sites; do
|
|
site_dest=$(echo $s | cut -d: -f1)
|
|
site_source=$(echo $s | cut -d: -f2)
|
|
force_flag=$(echo $s | cut -d: -f3)
|
|
mkdir -p vendor.copy/$site_dest
|
|
if [ -n "$force_flag" ]; then
|
|
echo "[INFO] $site_dest: force copying .go files"
|
|
rm -rf vendor.copy/$site_dest
|
|
rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest
|
|
else
|
|
[ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; }
|
|
fi
|
|
done
|
|
}
|
|
"""
|
|
sites = []
|
|
for module in self.modules_require:
|
|
# <dest>:<source>[:force]
|
|
if module in self.modules_srcpaths:
|
|
srcpath = self.modules_srcpaths[module]
|
|
logger.debug("Using %s as srcpath of module (%s)" % (srcpath, module))
|
|
else:
|
|
srcpath = module
|
|
sites.append("%s:%s:force" % (module, srcpath))
|
|
# To avoid the former one being overriden by the latter one when the former one is a subpath of the latter one, sort sites
|
|
sites.sort(key=len)
|
|
with open(relocation_inc_file, 'w') as f:
|
|
sites_str = ' \\\n '.join(sites)
|
|
f.write(template % sites_str)
|
|
logger.info("%s generated" % relocation_inc_file)
|
|
|
|
def gen_modules_txt(self):
|
|
"""
|
|
Generate modules.txt file
|
|
"""
|
|
modules_txt_file = os.path.join(self.workdir, 'modules.txt')
|
|
with open(modules_txt_file, 'w') as f:
|
|
for l in self.require_lines:
|
|
f.write('# %s\n' % l)
|
|
f.write('## explicit\n')
|
|
for l in self.replace_lines:
|
|
f.write('# %s\n' %l)
|
|
logger.info("%s generated" % modules_txt_file)
|
|
|
|
def sanity_check(self):
|
|
"""
|
|
Various anity checks
|
|
"""
|
|
sanity_check_ok = True
|
|
#
|
|
# Sanity Check 1:
|
|
# For modules having the same repo, at most one is allowed to not have subpath.
|
|
# This check operates on self.modules_repoinfo and self.modules_subpaths
|
|
#
|
|
repo_modules = {}
|
|
for module in self.modules_repoinfo:
|
|
# first form {repo: [module1, module2, ...]}
|
|
repo_url, repo_dest_dir, fullsrcrev = self.modules_repoinfo[module]
|
|
if repo_url not in repo_modules:
|
|
repo_modules[repo_url] = [module]
|
|
else:
|
|
repo_modules[repo_url].append(module)
|
|
for repo in repo_modules:
|
|
modules = repo_modules[repo]
|
|
if len(modules) == 1:
|
|
continue
|
|
# for modules sharing the same repo, at most one is allowed to not have subpath
|
|
nosubpath_modules = []
|
|
for m in modules:
|
|
if m not in self.modules_subpaths:
|
|
nosubpath_modules.append(m)
|
|
if len(nosubpath_modules) == 0:
|
|
continue
|
|
if len(nosubpath_modules) > 1:
|
|
logger.warning("Multiple modules sharing %s, but they don't have subpath: %s. Please double check." % (repo, nosubpath_modules))
|
|
if len(nosubpath_modules) == 1:
|
|
# do further check, OK if the module is the prefix for other modules sharing the same repo
|
|
module_to_check = nosubpath_modules[0]
|
|
for m in modules:
|
|
if module_to_check == m:
|
|
continue
|
|
if not m.startswith('%s/' % module_to_check):
|
|
logger.warning("%s is sharing repo (%s) with other modules, and it might need a subpath. Please double check: %s and: %s" % (module_to_check, repo, nosubpath_modules,m))
|
|
continue
|
|
|
|
#
|
|
# End of Sanity Check
|
|
#
|
|
if not sanity_check_ok:
|
|
sys.exit(1)
|
|
return
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="go mod dependency -> SRC_URI procesing",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog=textwrap.dedent('''\
|
|
|
|
Overview:
|
|
=========
|
|
|
|
go-mod-oe is a tool for processing go dependencies to generate
|
|
dependencies suitable for OE fetcher consumption.
|
|
|
|
In particular, it creates a build structure suitable for
|
|
'-mod="vendor"' go builds. Once complete all go mod dependencies
|
|
are in the vendor/ directory, so no golang specific fetching or
|
|
network access happens during the build.
|
|
|
|
The files src_uri.inc, relocation.inc and modules.txt are generated
|
|
and suitable for recipe inclusion.
|
|
|
|
A recipe build can then use these files to leverage the git fetcher
|
|
and related functionality (mirrors, sstate, etc).
|
|
|
|
Note 1: --rev does not have to be a tag, if you want to track the tip of
|
|
a branch specify the latest git has on that branch, and it will
|
|
be used.
|
|
|
|
Note 2: This script does not generate an entire recipe, the way the
|
|
the outputs are used can be modified as required.
|
|
|
|
Note 3: if a go.mod has a bad revision, or needs to be manually updated
|
|
to fetch fixes: go.mod in the main repository (see the repos/
|
|
directory). If go.mod is edited, modules.txt also has to be
|
|
updated to match the revision information.
|
|
|
|
Note 4: if an entry in go.mod is resolving to a destination that doesn't
|
|
have a SRCREV (i.e. golang.org vs github), the destination can
|
|
be temporarily overriden by editing: wget-contents/<repo>.repo_url.cache
|
|
The next run will use the cached value versus looking it up.
|
|
|
|
% vi wget-contents/golang.org_x_sys.repo_url.cache
|
|
|
|
How to use in a recipe:
|
|
=======================
|
|
|
|
There are examples in meta-virtualization of recipes that use this
|
|
script and stragegy for builds: docker-compose, nerdcli, k3s
|
|
|
|
1) The recipe should set the master repository SRCREV details, and then include
|
|
the src_uri.inc file:
|
|
|
|
SRCREV_nerdcli = "e084a2df4a8861eb5f0b0d32df0643ef24b81093"
|
|
SRC_URI = "git://github.com/containerd/nerdctl.git;name=nerdcli;branch=master;protocol=https"
|
|
|
|
include src_uri.inc
|
|
|
|
This results in the SRC_URI being fully populated with the main
|
|
repository and all dependencies.
|
|
|
|
2) The recipe should either copy, or include the relocation.inc file. It sets
|
|
a variable "sites" that is a list of source locations (where the src_uri.inc
|
|
fetches) and destination in a vendor directory, it also has a do_compile:prepend()
|
|
that contains a loop which relocates the fetches into a vendor.copy directory.
|
|
|
|
It is expected to be processed as follows, before compilation starts:
|
|
|
|
# sets the "sites" variable and copies files
|
|
include relocation.inc
|
|
|
|
The do_compile:prepend, contains the following loop:
|
|
|
|
cd ${S}/src/import
|
|
# this moves all the fetches into the proper vendor structure
|
|
# expected for build
|
|
for s in ${sites}; do
|
|
site_dest=$(echo $s | cut -d: -f1)
|
|
site_source=$(echo $s | cut -d: -f2)
|
|
force_flag=$(echo $s | cut -d: -f3)
|
|
mkdir -p vendor.copy/$site_dest
|
|
if [ -n "$force_flag" ]; then
|
|
echo "[INFO] $site_dest: force copying .go files"
|
|
rm -rf vendor.copy/$site_dest
|
|
rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest
|
|
else
|
|
[ -n "$(ls -A vendor.copy/$site_dest/*.go 2> /dev/null)" ] && { echo "[INFO] vendor.fetch/$site_source -> $site_dest: go copy skipped (files present)" ; true ; } || { echo "[INFO] $site_dest: copying .go files" ; rsync -a --exclude='vendor/' --exclude='.git/' vendor.fetch/$site_source/ vendor.copy/$site_dest ; }
|
|
fi
|
|
done
|
|
|
|
The main compile() function, should set the appropriate GO variables,
|
|
copy modules.txt and build the appripriate target:
|
|
|
|
# our copied .go files are to be used for the build
|
|
ln -sf vendor.copy vendor
|
|
|
|
3) The modules.txt file should be copied into the recipe directory, included
|
|
on the SRC_URI and copied into place after the relocation has been
|
|
processed.
|
|
|
|
# patches and config
|
|
SRC_URI += "file://0001-Makefile-allow-external-specification-of-build-setti.patch \\
|
|
file://modules.txt \
|
|
"
|
|
|
|
.....
|
|
|
|
cp ${WORKDIR}/modules.txt vendor/
|
|
|
|
Example: Updating the K3S recipe
|
|
================================
|
|
|
|
% cd meta-virtualization/recipe-containers/k3s/
|
|
# produces src_uri.inc, relocation.inc and modules.txt in the current directory
|
|
% ../../scripts/oe-go-mod-autogen.py --repo https://github.com/rancher/k3s.git --rev v1.27.5+k3s1
|
|
|
|
% cp modules.txt k3s/
|
|
|
|
... add and commit files.
|
|
|
|
|
|
'''))
|
|
parser.add_argument("--repo", help = "Repo for the recipe.", required=True)
|
|
parser.add_argument("--rev", help = "Revision for the recipe.", required=True)
|
|
parser.add_argument("--module", help = "Go module name. To be used with '--test'")
|
|
parser.add_argument("--version", help = "Go module version. To be used with '--test'")
|
|
parser.add_argument("--test", help = "Test to get repo url and fullsrcrev, used together with --module and --version.", action="store_true")
|
|
parser.add_argument("--workdir", help = "Working directory to hold intermediate results and output.", default=os.getcwd())
|
|
parser.add_argument("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_argument("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
parser.add_argument("-v", action='store_true', dest="verbose",
|
|
help="verbose")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose:
|
|
args.loglevel = args.verbose
|
|
args = parser.parse_args()
|
|
|
|
logger.setLevel(args.loglevel)
|
|
logger.debug("oe-go-mod-autogen.py running for %s:%s in %s" % (args.repo, args.rev, args.workdir))
|
|
gomodtool = GoModTool(args.repo, args.rev, args.workdir)
|
|
if args.test:
|
|
if not args.module or not args.version:
|
|
print("Please specify --module and --version")
|
|
sys.exit(1)
|
|
url, srcrev = gomodtool.get_url_srcrev(args.module, args.version)
|
|
print("url = %s, srcrev = %s" % (url, srcrev))
|
|
if not url or not srcrev:
|
|
print("Failed to get url & srcrev for %s:%s" % (args.module, args.version))
|
|
else:
|
|
gomodtool.parse()
|
|
gomodtool.sanity_check()
|
|
gomodtool.gen_src_uri_inc()
|
|
gomodtool.gen_relocation_inc()
|
|
gomodtool.gen_modules_txt()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
ret = main()
|
|
except Exception as esc:
|
|
ret = 1
|
|
import traceback
|
|
traceback.print_exc()
|
|
sys.exit(ret)
|