mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 20:59:01 +02:00
60fd912b44
For use the same code base for Recipe reporting system moves non-WEB code outside layerindex, create lib directory that contains common code for Layerindex and RRS. Create scripts directory that contains common infraestructure to populate db and tool scripts, modularize update.py script move Layerindex update to own folder. Update references for utils and recipeparse modules to point lib directory. Add missing GitPython into requirements.txt. Signed-off-by: Aníbal Limón <anibal.limon@linux.intel.com>
415 lines
16 KiB
Python
Executable File
415 lines
16 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Import a layer into the database
|
|
#
|
|
# Copyright (C) 2013 Intel Corporation
|
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
|
#
|
|
# Licensed under the MIT license, see COPYING.MIT for details
|
|
|
|
|
|
import sys
|
|
import os.path
|
|
|
|
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '../../lib')))
|
|
|
|
import optparse
|
|
import re
|
|
import glob
|
|
import utils
|
|
import logging
|
|
import subprocess
|
|
|
|
logger = utils.logger_create('LayerIndexImport')
|
|
|
|
link_re = re.compile(r'\[(http.*) +link\]')
|
|
|
|
def set_vcs_fields(layer, repoval):
|
|
layer.vcs_url = repoval
|
|
if repoval.startswith('git://git.openembedded.org/'):
|
|
reponame = re.sub('^.*/', '', repoval)
|
|
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
elif 'git.yoctoproject.org/' in repoval:
|
|
reponame = re.sub('^.*/', '', repoval)
|
|
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
|
|
elif 'github.com/' in repoval:
|
|
reponame = re.sub('^.*github.com/', '', repoval)
|
|
reponame = re.sub('.git$', '', reponame)
|
|
layer.vcs_web_url = 'http://github.com/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
|
|
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
|
|
elif 'gitorious.org/' in repoval:
|
|
reponame = re.sub('^.*gitorious.org/', '', repoval)
|
|
reponame = re.sub('.git$', '', reponame)
|
|
layer.vcs_web_url = 'http://gitorious.org/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
|
|
layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
|
|
elif 'bitbucket.org/' in repoval:
|
|
reponame = re.sub('^.*bitbucket.org/', '', repoval)
|
|
reponame = re.sub('.git$', '', reponame)
|
|
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
|
|
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
|
|
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
|
|
|
|
|
|
def readme_extract(readmefn):
|
|
maintainer_re = re.compile('maintaine[r(s)ed by]*[:\n\r]', re.IGNORECASE)
|
|
deps_re = re.compile('depend[sencies upon]*[:\n\r]', re.IGNORECASE)
|
|
|
|
maintlines = []
|
|
deps = []
|
|
desc = ''
|
|
maint_mode = False
|
|
blank_seen = False
|
|
deps_mode = False
|
|
desc_mode = True
|
|
with open(readmefn, 'r') as f:
|
|
for line in f.readlines():
|
|
if deps_mode:
|
|
if maintainer_re.search(line):
|
|
deps_mode = False
|
|
else:
|
|
if ':' in line:
|
|
blank_seen = False
|
|
if line.startswith('URI:'):
|
|
deps.append(line.split(':', 1)[-1].strip())
|
|
if line.startswith('layers:'):
|
|
deps[len(deps)-1] = (deps[len(deps)-1], line.split(':', 1)[-1].strip())
|
|
elif not (line.startswith('====') or line.startswith('----')):
|
|
if blank_seen:
|
|
deps_mode = False
|
|
else:
|
|
blank_seen = True
|
|
continue
|
|
|
|
if maint_mode:
|
|
line = line.strip()
|
|
if line and '@' in line or ' at ' in line:
|
|
maintlines.append(line)
|
|
elif not (line.startswith('====') or line.startswith('----')):
|
|
if maintlines or blank_seen:
|
|
maint_mode = False
|
|
else:
|
|
blank_seen = True
|
|
elif maintainer_re.search(line):
|
|
desc_mode = False
|
|
maint_mode = True
|
|
blank_seen = False
|
|
if ':' in line:
|
|
line = line.rsplit(":", 1)[-1].strip()
|
|
if line:
|
|
maintlines.append(line)
|
|
elif deps_re.search(line):
|
|
desc_mode = False
|
|
deps_mode = True
|
|
blank_seen = False
|
|
elif desc_mode:
|
|
if not line.strip():
|
|
if blank_seen:
|
|
desc_mode = False
|
|
blank_seen = True
|
|
elif line.startswith('====') or line.startswith('----'):
|
|
# Assume we just got the title, we don't need that
|
|
desc = ''
|
|
else:
|
|
desc += line
|
|
|
|
maintainers = []
|
|
for line in maintlines:
|
|
for maint in line.split(','):
|
|
if '@' in maint or ' at ' in maint and not 'yyyyyy@zzzzz.com' in maint:
|
|
maintainers.append(maint.strip())
|
|
return desc, maintainers, deps
|
|
|
|
|
|
def maintainers_extract(maintfn):
|
|
maintainers = []
|
|
with open(maintfn, 'r') as f:
|
|
for line in f.readlines():
|
|
if line.startswith('M:'):
|
|
line = line.split(':', 1)[-1].strip()
|
|
if line and '@' in line or ' at ' in line:
|
|
maintainers.append(line)
|
|
return list(set(maintainers))
|
|
|
|
|
|
def get_github_layerinfo(layer_url, username = None, password = None):
|
|
import httplib
|
|
import json
|
|
from layerindex.models import LayerMaintainer
|
|
|
|
def github_api_call(path):
|
|
conn = httplib.HTTPSConnection('api.github.com')
|
|
headers = {"User-Agent": "test_github.py"}
|
|
if username:
|
|
import base64
|
|
auth = base64.encodestring('%s:%s' % (username, password)).replace('\n', '')
|
|
headers['Authorization'] = "Basic %s" % auth
|
|
|
|
conn.request("GET", path, headers=headers)
|
|
resp = conn.getresponse()
|
|
return resp
|
|
|
|
json_data = None
|
|
owner_json_data = None
|
|
|
|
if layer_url.endswith('.git'):
|
|
layer_url = layer_url[:-4]
|
|
resp = github_api_call('/repos/%s' % layer_url.split('github.com/')[-1].rstrip('/'))
|
|
if resp.status in [200, 302]:
|
|
data = resp.read()
|
|
json_data = json.loads(data)
|
|
#headers = dict((key, value) for key, value in resp.getheaders())
|
|
#print headers
|
|
owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
|
|
if resp.status in [200, 302]:
|
|
owner_data = owner_resp.read()
|
|
owner_json_data = json.loads(owner_data)
|
|
else:
|
|
logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read()))
|
|
else:
|
|
logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read()))
|
|
|
|
return (json_data, owner_json_data)
|
|
|
|
|
|
def main():
|
|
parser = optparse.OptionParser(
|
|
usage = """
|
|
%prog [options] <url> [name]""")
|
|
|
|
parser.add_option("-s", "--subdir",
|
|
help = "Specify subdirectory",
|
|
action="store", dest="subdir")
|
|
parser.add_option("-n", "--dry-run",
|
|
help = "Don't write any data back to the database",
|
|
action="store_true", dest="dryrun")
|
|
parser.add_option("-d", "--debug",
|
|
help = "Enable debug output",
|
|
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
|
|
parser.add_option("", "--github-auth",
|
|
help = "Specify github username:password",
|
|
action="store", dest="github_auth")
|
|
parser.add_option("-q", "--quiet",
|
|
help = "Hide all output except error messages",
|
|
action="store_const", const=logging.ERROR, dest="loglevel")
|
|
|
|
options, args = parser.parse_args(sys.argv)
|
|
|
|
if len(args) < 2:
|
|
print("Please specify URL of repository for layer")
|
|
sys.exit(1)
|
|
|
|
layer_url = args[1]
|
|
|
|
if len(args) > 2:
|
|
layer_name = args[2]
|
|
else:
|
|
if options.subdir:
|
|
layer_name = options.subdir
|
|
else:
|
|
layer_name = filter(None, layer_url.split('/'))[-1]
|
|
if layer_name.endswith('.git'):
|
|
layer_name = layer_name[:-4]
|
|
|
|
if options.github_auth:
|
|
if not ':' in options.github_auth:
|
|
logger.error('--github-auth value must be specified as username:password')
|
|
sys.exit(1)
|
|
splitval = options.github_auth.split(':')
|
|
github_login = splitval[0]
|
|
github_password = splitval[1]
|
|
else:
|
|
github_login = None
|
|
github_password = None
|
|
|
|
utils.setup_django()
|
|
import settings
|
|
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
|
|
from django.db import transaction
|
|
|
|
logger.setLevel(options.loglevel)
|
|
|
|
fetchdir = settings.LAYER_FETCH_DIR
|
|
if not fetchdir:
|
|
logger.error("Please set LAYER_FETCH_DIR in settings.py")
|
|
sys.exit(1)
|
|
|
|
if not os.path.exists(fetchdir):
|
|
os.makedirs(fetchdir)
|
|
|
|
master_branch = utils.get_branch('master')
|
|
core_layer = None
|
|
transaction.enter_transaction_management()
|
|
transaction.managed(True)
|
|
try:
|
|
# Fetch layer
|
|
logger.info('Fetching repository %s' % layer_url)
|
|
|
|
layer = LayerItem()
|
|
layer.name = layer_name
|
|
layer.status = 'P'
|
|
layer.layer_type = 'M'
|
|
layer.summary = 'tempvalue'
|
|
layer.description = layer.summary
|
|
|
|
set_vcs_fields(layer, layer_url)
|
|
|
|
urldir = layer.get_fetch_dir()
|
|
repodir = os.path.join(fetchdir, urldir)
|
|
out = None
|
|
try:
|
|
if not os.path.exists(repodir):
|
|
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
|
|
else:
|
|
out = utils.runcmd("git fetch", repodir, logger=logger)
|
|
except Exception as e:
|
|
logger.error("Fetch failed: %s" % str(e))
|
|
sys.exit(1)
|
|
|
|
actual_branch = ''
|
|
try:
|
|
out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
|
|
except subprocess.CalledProcessError:
|
|
branches = utils.runcmd("git branch -r", repodir, logger=logger)
|
|
for line in branches.splitlines():
|
|
if 'origin/HEAD ->' in line:
|
|
actual_branch = line.split('-> origin/')[-1]
|
|
break
|
|
if not actual_branch:
|
|
logger.error("Repository has no master branch nor origin/HEAD")
|
|
sys.exit(1)
|
|
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
|
|
|
|
layer_paths = []
|
|
if options.subdir:
|
|
layerdir = os.path.join(repodir, options.subdir)
|
|
if not os.path.exists(layerdir):
|
|
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
|
|
sys.exit(1)
|
|
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
|
|
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
|
|
sys.exit(1)
|
|
layer_paths.append(layerdir)
|
|
else:
|
|
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
|
|
layer_paths.append(repodir)
|
|
# Find subdirs with a conf/layer.conf
|
|
for subdir in os.listdir(repodir):
|
|
subdir_path = os.path.join(repodir, subdir)
|
|
if os.path.isdir(subdir_path):
|
|
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
|
|
layer_paths.append(subdir_path)
|
|
if not layer_paths:
|
|
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
|
|
sys.exit(1)
|
|
|
|
if 'github.com' in layer.vcs_url:
|
|
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
|
|
|
|
for layerdir in layer_paths:
|
|
layer.pk = None
|
|
if layerdir != repodir:
|
|
subdir = os.path.relpath(layerdir, repodir)
|
|
if len(layer_paths) > 1:
|
|
layer.name = subdir
|
|
else:
|
|
subdir = ''
|
|
if LayerItem.objects.filter(name=layer.name).exists():
|
|
logger.error('A layer named "%s" already exists in the database' % layer_name)
|
|
sys.exit(1)
|
|
|
|
logger.info('Creating layer %s' % layer.name)
|
|
# Guess layer type
|
|
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
|
|
layer.layer_type = 'D'
|
|
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
|
|
layer.layer_type = 'B'
|
|
layer.save()
|
|
layerbranch = LayerBranch()
|
|
layerbranch.layer = layer
|
|
layerbranch.branch = master_branch
|
|
if layerdir != repodir:
|
|
layerbranch.vcs_subdir = subdir
|
|
if actual_branch:
|
|
layerbranch.actual_branch = actual_branch
|
|
layerbranch.save()
|
|
if layer.name != settings.CORE_LAYER_NAME:
|
|
if not core_layer:
|
|
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
|
|
if core_layer:
|
|
layerdep = LayerDependency()
|
|
layerdep.layerbranch = layerbranch
|
|
layerdep.dependency = core_layer
|
|
layerdep.save()
|
|
|
|
# Get some extra meta-information
|
|
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
|
|
if (not readme_files) and subdir:
|
|
readme_files = glob.glob(os.path.join(repodir, 'README*'))
|
|
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
|
|
if (not maintainer_files) and subdir:
|
|
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
|
|
|
|
maintainers = []
|
|
if readme_files:
|
|
(desc, maintainers, deps) = readme_extract(readme_files[0])
|
|
if desc:
|
|
layer.summary = layer.name
|
|
layer.description = desc
|
|
if maintainer_files:
|
|
maintainers.extend(maintainers_extract(readme_files[0]))
|
|
|
|
if (not maintainers) and 'github.com' in layer.vcs_url:
|
|
if json_data:
|
|
layer.summary = json_data['description']
|
|
layer.description = layer.summary
|
|
if owner_json_data:
|
|
owner_name = owner_json_data.get('name', None)
|
|
owner_email = owner_json_data.get('email', None)
|
|
if owner_name and owner_email:
|
|
maintainers.append('%s <%s>' % (owner_name, owner_email))
|
|
|
|
if layer.name == 'openembedded-core':
|
|
layer.summary = 'Core metadata'
|
|
layer.layer_type = 'A'
|
|
elif layer.name == 'meta-oe':
|
|
layer.summary = 'Additional shared OE metadata'
|
|
layer.description = layer.summary
|
|
layer.layer_type = 'A'
|
|
|
|
if maintainers:
|
|
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
|
|
for maintentry in maintainers:
|
|
res = maint_re.match(maintentry)
|
|
if res:
|
|
maintainer = LayerMaintainer()
|
|
maintainer.layerbranch = layerbranch
|
|
maintainer.name = res.group(1).strip()
|
|
maintainer.email = res.group(2)
|
|
if res.group(3):
|
|
maintainer.responsibility = res.group(3).strip()
|
|
maintainer.save()
|
|
|
|
layer.save()
|
|
|
|
if options.dryrun:
|
|
transaction.rollback()
|
|
else:
|
|
transaction.commit()
|
|
except:
|
|
transaction.rollback()
|
|
raise
|
|
finally:
|
|
transaction.leave_transaction_management()
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|