mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-05 21:24:46 +02:00
752e0152c2
Subdirectories are scanned when adding layers. If any of the subdirectories or root directory layers already exist in the database, then the addition fails. This changes that behaviour to report the failure as a warning and remove it from the list. That way, if a repo has a new layer added it can be rescanned without issue. Layers being rescanned are checked against the vcs_url to ensure there is not a name collision. A name collision without the same vcs_url will still produce a hard failure. Note that multiple layers with the same vcs_url are supported in the error reporting even though this should never happen. Signed-off-by: Liam R. Howlett <Liam.Howlett@WindRiver.com>
440 lines
18 KiB
Python
Executable File
440 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Import a layer into the database
|
|
#
|
|
# Copyright (C) 2016 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
|
|
import sys
|
|
import os.path
|
|
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
|
|
|
|
import optparse
|
|
import re
|
|
import glob
|
|
import utils
|
|
import logging
|
|
import subprocess
|
|
|
|
class DryRunRollbackException(Exception):
|
|
pass
|
|
|
|
logger = utils.logger_create('LayerIndexImport')
|
|
|
|
link_re = re.compile(r'\[(http.*) +link\]')
|
|
|
|
def set_vcs_fields(layer, repoval):
|
|
layer.vcs_url = repoval
|
|
if repoval.startswith('git://git.openembedded.org/'):
|
|
reponame = re.sub('^.*/', '', repoval)
|
|
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
elif 'git.yoctoproject.org/' in repoval:
|
|
reponame = re.sub('^.*/', '', repoval)
|
|
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
elif 'github.com/' in repoval:
|
|
reponame = re.sub('^.*github.com/', '', repoval)
|
|
reponame = re.sub('.git$', '', reponame)
|
|
layer.vcs_web_url = 'http://github.com/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
|
|
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
|
|
elif 'gitorious.org/' in repoval:
|
|
reponame = re.sub('^.*gitorious.org/', '', repoval)
|
|
reponame = re.sub('.git$', '', reponame)
|
|
layer.vcs_web_url = 'http://gitorious.org/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
|
|
layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
|
|
elif 'bitbucket.org/' in repoval:
|
|
reponame = re.sub('^.*bitbucket.org/', '', repoval)
|
|
reponame = re.sub('.git$', '', reponame)
|
|
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
|
|
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
|
|
|
|
|
|
def readme_extract(readmefn):
|
|
maintainer_re = re.compile('maintaine[r(s)ed by]*[:\n\r]', re.IGNORECASE)
|
|
deps_re = re.compile('depend[sencies upon]*[:\n\r]', re.IGNORECASE)
|
|
|
|
maintlines = []
|
|
deps = []
|
|
desc = ''
|
|
maint_mode = False
|
|
blank_seen = False
|
|
deps_mode = False
|
|
desc_mode = True
|
|
with open(readmefn, 'r') as f:
|
|
for line in f.readlines():
|
|
if deps_mode:
|
|
if maintainer_re.search(line):
|
|
deps_mode = False
|
|
else:
|
|
if ':' in line:
|
|
blank_seen = False
|
|
if line.startswith('URI:'):
|
|
deps.append(line.split(':', 1)[-1].strip())
|
|
if line.startswith('layers:'):
|
|
deps[len(deps)-1] = (deps[len(deps)-1], line.split(':', 1)[-1].strip())
|
|
elif not (line.startswith('====') or line.startswith('----')):
|
|
if blank_seen:
|
|
deps_mode = False
|
|
else:
|
|
blank_seen = True
|
|
continue
|
|
|
|
if maint_mode:
|
|
line = line.strip()
|
|
if line and '@' in line or ' at ' in line:
|
|
maintlines.append(line)
|
|
elif not (line.startswith('====') or line.startswith('----')):
|
|
if maintlines or blank_seen:
|
|
maint_mode = False
|
|
else:
|
|
blank_seen = True
|
|
elif maintainer_re.search(line):
|
|
desc_mode = False
|
|
maint_mode = True
|
|
blank_seen = False
|
|
if ':' in line:
|
|
line = line.rsplit(":", 1)[-1].strip()
|
|
if line:
|
|
maintlines.append(line)
|
|
elif deps_re.search(line):
|
|
desc_mode = False
|
|
deps_mode = True
|
|
blank_seen = False
|
|
elif desc_mode:
|
|
if not line.strip():
|
|
if blank_seen:
|
|
desc_mode = False
|
|
blank_seen = True
|
|
elif line.startswith('====') or line.startswith('----'):
|
|
# Assume we just got the title, we don't need that
|
|
desc = ''
|
|
else:
|
|
desc += line
|
|
|
|
maintainers = []
|
|
for line in maintlines:
|
|
for maint in line.split(','):
|
|
if '@' in maint or ' at ' in maint and not 'yyyyyy@zzzzz.com' in maint:
|
|
maintainers.append(maint.strip())
|
|
return desc, maintainers, deps
|
|
|
|
|
|
def maintainers_extract(maintfn):
|
|
maintainers = []
|
|
with open(maintfn, 'r') as f:
|
|
for line in f.readlines():
|
|
if line.startswith('M:'):
|
|
line = line.split(':', 1)[-1].strip()
|
|
if line and '@' in line or ' at ' in line:
|
|
maintainers.append(line)
|
|
return list(set(maintainers))
|
|
|
|
|
|
def get_github_layerinfo(layer_url, username = None, password = None):
|
|
import http.client
|
|
import json
|
|
from layerindex.models import LayerMaintainer
|
|
|
|
def github_api_call(path):
|
|
conn = http.client.HTTPSConnection('api.github.com')
|
|
headers = {"User-Agent": "test_github.py"}
|
|
if username:
|
|
import base64
|
|
auth = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
|
|
headers['Authorization'] = "Basic %s" % auth
|
|
|
|
conn.request("GET", path, headers=headers)
|
|
resp = conn.getresponse()
|
|
return resp
|
|
|
|
json_data = None
|
|
owner_json_data = None
|
|
|
|
if layer_url.endswith('.git'):
|
|
layer_url = layer_url[:-4]
|
|
resp = github_api_call('/repos/%s' % layer_url.split('github.com/')[-1].rstrip('/'))
|
|
if resp.status in [200, 302]:
|
|
data = resp.read().decode('utf-8')
|
|
json_data = json.loads(data)
|
|
#headers = dict((key, value) for key, value in resp.getheaders())
|
|
#print(headers)
|
|
owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
|
|
if resp.status in [200, 302]:
|
|
owner_data = owner_resp.read().decode('utf-8')
|
|
owner_json_data = json.loads(owner_data)
|
|
else:
|
|
logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read().decode('utf-8')))
|
|
else:
|
|
logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read().decode('utf-8')))
|
|
|
|
return (json_data, owner_json_data)
|
|
|
|
|
|
def main():
|
|
valid_layer_name = re.compile('[-\w]+$')
|
|
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options] <url> [name]""")
|
|
|
|
parser.add_option("-s", "--subdir",
|
|
help = "Specify subdirectory",
|
|
action="store", dest="subdir")
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("", "--github-auth",
|
|
help = "Specify github username:password",
|
|
action="store", dest="github_auth")
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
parser.add_option("-a", "--actual-branch",
|
|
help = "Set actual branch",
|
|
action="store", dest="actual_branch")
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
if len(args) < 2:
|
|
print("Please specify URL of repository for layer")
|
|
sys.exit(1)
|
|
|
|
layer_url = args[1]
|
|
|
|
if len(args) > 2:
|
|
layer_name = args[2]
|
|
else:
|
|
if options.subdir:
|
|
layer_name = options.subdir
|
|
else:
|
|
layer_name = [x for x in layer_url.split('/') if x][-1]
|
|
if layer_name.endswith('.git'):
|
|
layer_name = layer_name[:-4]
|
|
|
|
if not valid_layer_name.match(layer_name):
|
|
logger.error('Invalid layer name "%s" - Layer name can only include letters, numbers and dashes.', layer_name)
|
|
sys.exit(1)
|
|
|
|
if options.github_auth:
|
|
if not ':' in options.github_auth:
|
|
logger.error('--github-auth value must be specified as username:password')
|
|
sys.exit(1)
|
|
splitval = options.github_auth.split(':')
|
|
github_login = splitval[0]
|
|
github_password = splitval[1]
|
|
else:
|
|
github_login = None
|
|
github_password = None
|
|
|
|
utils.setup_django()
|
|
import settings
|
|
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
|
|
from django.db import transaction
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
fetchdir = settings.LAYER_FETCH_DIR
|
|
if not fetchdir:
|
|
logger.error("Please set LAYER_FETCH_DIR in settings.py")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(fetchdir):
|
|
os.makedirs(fetchdir)
|
|
|
|
master_branch = utils.get_branch('master')
|
|
core_layer = None
|
|
try:
|
|
with transaction.atomic():
|
|
# Fetch layer
|
|
logger.info('Fetching repository %s' % layer_url)
|
|
|
|
layer = LayerItem()
|
|
layer.name = layer_name
|
|
layer.status = 'P'
|
|
layer.layer_type = 'M'
|
|
layer.summary = 'tempvalue'
|
|
layer.description = layer.summary
|
|
|
|
set_vcs_fields(layer, layer_url)
|
|
|
|
urldir = layer.get_fetch_dir()
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
out = None
|
|
try:
|
|
if not os.path.exists(repodir):
|
|
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
|
|
else:
|
|
out = utils.runcmd("git fetch", repodir, logger=logger)
|
|
except Exception as e:
|
|
logger.error("Fetch failed: %s" % str(e))
|
|
sys.exit(1)
|
|
|
|
actual_branch = 'master'
|
|
if (options.actual_branch):
|
|
actual_branch = options.actual_branch
|
|
try:
|
|
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
|
|
except subprocess.CalledProcessError:
|
|
actual_branch = None
|
|
branches = utils.runcmd("git branch -r", repodir, logger=logger)
|
|
for line in branches.splitlines():
|
|
if 'origin/HEAD ->' in line:
|
|
actual_branch = line.split('-> origin/')[-1]
|
|
break
|
|
if not actual_branch:
|
|
logger.error("Repository has no master branch nor origin/HEAD")
|
|
sys.exit(1)
|
|
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
|
|
|
|
layer_paths = []
|
|
if options.subdir:
|
|
layerdir = os.path.join(repodir, options.subdir)
|
|
if not os.path.exists(layerdir):
|
|
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
|
|
sys.exit(1)
|
|
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
|
|
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
|
|
sys.exit(1)
|
|
layer_paths.append(layerdir)
|
|
else:
|
|
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
|
|
layer_paths.append(repodir)
|
|
# Find subdirs with a conf/layer.conf
|
|
for subdir in os.listdir(repodir):
|
|
subdir_path = os.path.join(repodir, subdir)
|
|
if os.path.isdir(subdir_path):
|
|
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
|
|
layer_paths.append(subdir_path)
|
|
if not layer_paths:
|
|
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
|
|
sys.exit(1)
|
|
|
|
if 'github.com' in layer.vcs_url:
|
|
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
|
|
|
|
for layerdir in layer_paths:
|
|
layer.pk = None
|
|
if layerdir != repodir:
|
|
subdir = os.path.relpath(layerdir, repodir)
|
|
if len(layer_paths) > 1:
|
|
layer.name = subdir
|
|
else:
|
|
subdir = ''
|
|
if LayerItem.objects.filter(name=layer.name).exists():
|
|
if LayerItem.objects.filter(name=layer.name).exclude(vcs_url=layer.vcs_url).exists():
|
|
conflict_list = LayerItem.objects.filter(name=layer.name).exclude(vcs_url=layer.vcs_url)
|
|
conflict_list_urls = []
|
|
for conflict in conflict_list:
|
|
conflict_list_urls.append(conflict.vcs_url)
|
|
cln = ', '.join(conflict_list_urls)
|
|
logger.error('A layer named "%s" already exists in the database. Possible name collision with %s.vcs_url = %s' % (layer.name, layer.name, cln))
|
|
sys.exit(1)
|
|
else:
|
|
logger.info('The layer named "%s" already exists in the database. Skipping this layer with same vcs_url' % layer.name)
|
|
layer_paths = [x for x in layer_paths if x != layerdir]
|
|
continue
|
|
|
|
|
|
|
|
logger.info('Creating layer %s' % layer.name)
|
|
# Guess layer type
|
|
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
|
|
layer.layer_type = 'D'
|
|
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
|
|
layer.layer_type = 'B'
|
|
layer.save()
|
|
layerbranch = LayerBranch()
|
|
layerbranch.layer = layer
|
|
layerbranch.branch = master_branch
|
|
if layerdir != repodir:
|
|
layerbranch.vcs_subdir = subdir
|
|
if actual_branch:
|
|
layerbranch.actual_branch = actual_branch
|
|
layerbranch.save()
|
|
if layer.name != settings.CORE_LAYER_NAME:
|
|
if not core_layer:
|
|
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
|
|
if core_layer:
|
|
layerdep = LayerDependency()
|
|
layerdep.layerbranch = layerbranch
|
|
layerdep.dependency = core_layer
|
|
layerdep.save()
|
|
|
|
# Get some extra meta-information
|
|
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
|
|
if (not readme_files) and subdir:
|
|
readme_files = glob.glob(os.path.join(repodir, 'README*'))
|
|
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
|
|
if (not maintainer_files) and subdir:
|
|
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
|
|
|
|
maintainers = []
|
|
if readme_files:
|
|
(desc, maintainers, deps) = readme_extract(readme_files[0])
|
|
if desc:
|
|
layer.summary = layer.name
|
|
layer.description = desc
|
|
if maintainer_files:
|
|
maintainers.extend(maintainers_extract(readme_files[0]))
|
|
|
|
if (not maintainers) and 'github.com' in layer.vcs_url:
|
|
if json_data:
|
|
layer.summary = json_data['description']
|
|
layer.description = layer.summary
|
|
if owner_json_data:
|
|
owner_name = owner_json_data.get('name', None)
|
|
owner_email = owner_json_data.get('email', None)
|
|
if owner_name and owner_email:
|
|
maintainers.append('%s <%s>' % (owner_name, owner_email))
|
|
|
|
if layer.name == 'openembedded-core':
|
|
layer.summary = 'Core metadata'
|
|
layer.layer_type = 'A'
|
|
elif layer.name == 'meta-oe':
|
|
layer.summary = 'Additional shared OE metadata'
|
|
layer.description = layer.summary
|
|
layer.layer_type = 'A'
|
|
|
|
if maintainers:
|
|
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
|
|
for maintentry in maintainers:
|
|
res = maint_re.match(maintentry)
|
|
if res:
|
|
maintainer = LayerMaintainer()
|
|
maintainer.layerbranch = layerbranch
|
|
maintainer.name = res.group(1).strip()
|
|
maintainer.email = res.group(2)
|
|
if res.group(3):
|
|
maintainer.responsibility = res.group(3).strip()
|
|
maintainer.save()
|
|
|
|
layer.save()
|
|
|
|
if not layer_paths:
|
|
logger.error('No layers added.')
|
|
sys.exit(1);
|
|
|
|
if options.dryrun:
|
|
raise DryRunRollbackException()
|
|
except DryRunRollbackException:
|
|
pass
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|