mirror of
git://git.yoctoproject.org/layerindex-web.git
synced 2025-07-19 20:59:01 +02:00
Fix import script for Django 1.8 & Python 3
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton
parent
f268a3cfdb
commit
78406e520a
|
|
@ -1,8 +1,8 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
# Import a layer into the database
|
# Import a layer into the database
|
||||||
#
|
#
|
||||||
# Copyright (C) 2013 Intel Corporation
|
# Copyright (C) 2016 Intel Corporation
|
||||||
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
|
||||||
#
|
#
|
||||||
# Licensed under the MIT license, see COPYING.MIT for details
|
# Licensed under the MIT license, see COPYING.MIT for details
|
||||||
|
|
@ -20,6 +20,9 @@ import utils
|
||||||
import logging
|
import logging
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
class DryRunRollbackException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
logger = utils.logger_create('LayerIndexImport')
|
logger = utils.logger_create('LayerIndexImport')
|
||||||
|
|
||||||
link_re = re.compile(r'\[(http.*) +link\]')
|
link_re = re.compile(r'\[(http.*) +link\]')
|
||||||
|
|
@ -138,12 +141,12 @@ def maintainers_extract(maintfn):
|
||||||
|
|
||||||
|
|
||||||
def get_github_layerinfo(layer_url, username = None, password = None):
|
def get_github_layerinfo(layer_url, username = None, password = None):
|
||||||
import httplib
|
import http.client
|
||||||
import json
|
import json
|
||||||
from layerindex.models import LayerMaintainer
|
from layerindex.models import LayerMaintainer
|
||||||
|
|
||||||
def github_api_call(path):
|
def github_api_call(path):
|
||||||
conn = httplib.HTTPSConnection('api.github.com')
|
conn = http.client.HTTPSConnection('api.github.com')
|
||||||
headers = {"User-Agent": "test_github.py"}
|
headers = {"User-Agent": "test_github.py"}
|
||||||
if username:
|
if username:
|
||||||
import base64
|
import base64
|
||||||
|
|
@ -161,18 +164,18 @@ def get_github_layerinfo(layer_url, username = None, password = None):
|
||||||
layer_url = layer_url[:-4]
|
layer_url = layer_url[:-4]
|
||||||
resp = github_api_call('/repos/%s' % layer_url.split('github.com/')[-1].rstrip('/'))
|
resp = github_api_call('/repos/%s' % layer_url.split('github.com/')[-1].rstrip('/'))
|
||||||
if resp.status in [200, 302]:
|
if resp.status in [200, 302]:
|
||||||
data = resp.read()
|
data = resp.read().decode('utf-8')
|
||||||
json_data = json.loads(data)
|
json_data = json.loads(data)
|
||||||
#headers = dict((key, value) for key, value in resp.getheaders())
|
#headers = dict((key, value) for key, value in resp.getheaders())
|
||||||
#print(headers)
|
#print(headers)
|
||||||
owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
|
owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
|
||||||
if resp.status in [200, 302]:
|
if resp.status in [200, 302]:
|
||||||
owner_data = owner_resp.read()
|
owner_data = owner_resp.read().decode('utf-8')
|
||||||
owner_json_data = json.loads(owner_data)
|
owner_json_data = json.loads(owner_data)
|
||||||
else:
|
else:
|
||||||
logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read()))
|
logger.error('HTTP status %s reading owner info from github API: %s' % (resp.status, resp.read().decode('utf-8')))
|
||||||
else:
|
else:
|
||||||
logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read()))
|
logger.error('HTTP status %s reading repo info from github API: %s' % (resp.status, resp.read().decode('utf-8')))
|
||||||
|
|
||||||
return (json_data, owner_json_data)
|
return (json_data, owner_json_data)
|
||||||
|
|
||||||
|
|
@ -212,7 +215,7 @@ def main():
|
||||||
if options.subdir:
|
if options.subdir:
|
||||||
layer_name = options.subdir
|
layer_name = options.subdir
|
||||||
else:
|
else:
|
||||||
layer_name = filter(None, layer_url.split('/'))[-1]
|
layer_name = [x for x in layer_url.split('/') if x][-1]
|
||||||
if layer_name.endswith('.git'):
|
if layer_name.endswith('.git'):
|
||||||
layer_name = layer_name[:-4]
|
layer_name = layer_name[:-4]
|
||||||
|
|
||||||
|
|
@ -244,168 +247,162 @@ def main():
|
||||||
|
|
||||||
master_branch = utils.get_branch('master')
|
master_branch = utils.get_branch('master')
|
||||||
core_layer = None
|
core_layer = None
|
||||||
transaction.enter_transaction_management()
|
|
||||||
transaction.managed(True)
|
|
||||||
try:
|
try:
|
||||||
# Fetch layer
|
with transaction.atomic():
|
||||||
logger.info('Fetching repository %s' % layer_url)
|
# Fetch layer
|
||||||
|
logger.info('Fetching repository %s' % layer_url)
|
||||||
|
|
||||||
layer = LayerItem()
|
layer = LayerItem()
|
||||||
layer.name = layer_name
|
layer.name = layer_name
|
||||||
layer.status = 'P'
|
layer.status = 'P'
|
||||||
layer.layer_type = 'M'
|
layer.layer_type = 'M'
|
||||||
layer.summary = 'tempvalue'
|
layer.summary = 'tempvalue'
|
||||||
layer.description = layer.summary
|
layer.description = layer.summary
|
||||||
|
|
||||||
set_vcs_fields(layer, layer_url)
|
set_vcs_fields(layer, layer_url)
|
||||||
|
|
||||||
urldir = layer.get_fetch_dir()
|
urldir = layer.get_fetch_dir()
|
||||||
repodir = os.path.join(fetchdir, urldir)
|
repodir = os.path.join(fetchdir, urldir)
|
||||||
out = None
|
out = None
|
||||||
try:
|
try:
|
||||||
if not os.path.exists(repodir):
|
if not os.path.exists(repodir):
|
||||||
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
|
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
|
||||||
|
else:
|
||||||
|
out = utils.runcmd("git fetch", repodir, logger=logger)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("Fetch failed: %s" % str(e))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
actual_branch = ''
|
||||||
|
try:
|
||||||
|
out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
branches = utils.runcmd("git branch -r", repodir, logger=logger)
|
||||||
|
for line in branches.splitlines():
|
||||||
|
if 'origin/HEAD ->' in line:
|
||||||
|
actual_branch = line.split('-> origin/')[-1]
|
||||||
|
break
|
||||||
|
if not actual_branch:
|
||||||
|
logger.error("Repository has no master branch nor origin/HEAD")
|
||||||
|
sys.exit(1)
|
||||||
|
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
|
||||||
|
|
||||||
|
layer_paths = []
|
||||||
|
if options.subdir:
|
||||||
|
layerdir = os.path.join(repodir, options.subdir)
|
||||||
|
if not os.path.exists(layerdir):
|
||||||
|
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
|
||||||
|
sys.exit(1)
|
||||||
|
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
|
||||||
|
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
|
||||||
|
sys.exit(1)
|
||||||
|
layer_paths.append(layerdir)
|
||||||
else:
|
else:
|
||||||
out = utils.runcmd("git fetch", repodir, logger=logger)
|
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
|
||||||
except Exception as e:
|
layer_paths.append(repodir)
|
||||||
logger.error("Fetch failed: %s" % str(e))
|
# Find subdirs with a conf/layer.conf
|
||||||
sys.exit(1)
|
for subdir in os.listdir(repodir):
|
||||||
|
subdir_path = os.path.join(repodir, subdir)
|
||||||
|
if os.path.isdir(subdir_path):
|
||||||
|
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
|
||||||
|
layer_paths.append(subdir_path)
|
||||||
|
if not layer_paths:
|
||||||
|
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
actual_branch = ''
|
if 'github.com' in layer.vcs_url:
|
||||||
try:
|
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
|
||||||
out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
branches = utils.runcmd("git branch -r", repodir, logger=logger)
|
|
||||||
for line in branches.splitlines():
|
|
||||||
if 'origin/HEAD ->' in line:
|
|
||||||
actual_branch = line.split('-> origin/')[-1]
|
|
||||||
break
|
|
||||||
if not actual_branch:
|
|
||||||
logger.error("Repository has no master branch nor origin/HEAD")
|
|
||||||
sys.exit(1)
|
|
||||||
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
|
|
||||||
|
|
||||||
layer_paths = []
|
for layerdir in layer_paths:
|
||||||
if options.subdir:
|
layer.pk = None
|
||||||
layerdir = os.path.join(repodir, options.subdir)
|
if layerdir != repodir:
|
||||||
if not os.path.exists(layerdir):
|
subdir = os.path.relpath(layerdir, repodir)
|
||||||
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
|
if len(layer_paths) > 1:
|
||||||
sys.exit(1)
|
layer.name = subdir
|
||||||
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
|
else:
|
||||||
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
|
subdir = ''
|
||||||
sys.exit(1)
|
if LayerItem.objects.filter(name=layer.name).exists():
|
||||||
layer_paths.append(layerdir)
|
logger.error('A layer named "%s" already exists in the database' % layer_name)
|
||||||
else:
|
sys.exit(1)
|
||||||
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
|
|
||||||
layer_paths.append(repodir)
|
|
||||||
# Find subdirs with a conf/layer.conf
|
|
||||||
for subdir in os.listdir(repodir):
|
|
||||||
subdir_path = os.path.join(repodir, subdir)
|
|
||||||
if os.path.isdir(subdir_path):
|
|
||||||
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
|
|
||||||
layer_paths.append(subdir_path)
|
|
||||||
if not layer_paths:
|
|
||||||
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if 'github.com' in layer.vcs_url:
|
logger.info('Creating layer %s' % layer.name)
|
||||||
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
|
# Guess layer type
|
||||||
|
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
|
||||||
|
layer.layer_type = 'D'
|
||||||
|
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
|
||||||
|
layer.layer_type = 'B'
|
||||||
|
layer.save()
|
||||||
|
layerbranch = LayerBranch()
|
||||||
|
layerbranch.layer = layer
|
||||||
|
layerbranch.branch = master_branch
|
||||||
|
if layerdir != repodir:
|
||||||
|
layerbranch.vcs_subdir = subdir
|
||||||
|
if actual_branch:
|
||||||
|
layerbranch.actual_branch = actual_branch
|
||||||
|
layerbranch.save()
|
||||||
|
if layer.name != settings.CORE_LAYER_NAME:
|
||||||
|
if not core_layer:
|
||||||
|
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
|
||||||
|
if core_layer:
|
||||||
|
layerdep = LayerDependency()
|
||||||
|
layerdep.layerbranch = layerbranch
|
||||||
|
layerdep.dependency = core_layer
|
||||||
|
layerdep.save()
|
||||||
|
|
||||||
for layerdir in layer_paths:
|
# Get some extra meta-information
|
||||||
layer.pk = None
|
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
|
||||||
if layerdir != repodir:
|
if (not readme_files) and subdir:
|
||||||
subdir = os.path.relpath(layerdir, repodir)
|
readme_files = glob.glob(os.path.join(repodir, 'README*'))
|
||||||
if len(layer_paths) > 1:
|
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
|
||||||
layer.name = subdir
|
if (not maintainer_files) and subdir:
|
||||||
else:
|
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
|
||||||
subdir = ''
|
|
||||||
if LayerItem.objects.filter(name=layer.name).exists():
|
|
||||||
logger.error('A layer named "%s" already exists in the database' % layer_name)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
logger.info('Creating layer %s' % layer.name)
|
maintainers = []
|
||||||
# Guess layer type
|
if readme_files:
|
||||||
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
|
(desc, maintainers, deps) = readme_extract(readme_files[0])
|
||||||
layer.layer_type = 'D'
|
if desc:
|
||||||
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
|
layer.summary = layer.name
|
||||||
layer.layer_type = 'B'
|
layer.description = desc
|
||||||
layer.save()
|
if maintainer_files:
|
||||||
layerbranch = LayerBranch()
|
maintainers.extend(maintainers_extract(readme_files[0]))
|
||||||
layerbranch.layer = layer
|
|
||||||
layerbranch.branch = master_branch
|
|
||||||
if layerdir != repodir:
|
|
||||||
layerbranch.vcs_subdir = subdir
|
|
||||||
if actual_branch:
|
|
||||||
layerbranch.actual_branch = actual_branch
|
|
||||||
layerbranch.save()
|
|
||||||
if layer.name != settings.CORE_LAYER_NAME:
|
|
||||||
if not core_layer:
|
|
||||||
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
|
|
||||||
if core_layer:
|
|
||||||
layerdep = LayerDependency()
|
|
||||||
layerdep.layerbranch = layerbranch
|
|
||||||
layerdep.dependency = core_layer
|
|
||||||
layerdep.save()
|
|
||||||
|
|
||||||
# Get some extra meta-information
|
if (not maintainers) and 'github.com' in layer.vcs_url:
|
||||||
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
|
if json_data:
|
||||||
if (not readme_files) and subdir:
|
layer.summary = json_data['description']
|
||||||
readme_files = glob.glob(os.path.join(repodir, 'README*'))
|
layer.description = layer.summary
|
||||||
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
|
if owner_json_data:
|
||||||
if (not maintainer_files) and subdir:
|
owner_name = owner_json_data.get('name', None)
|
||||||
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
|
owner_email = owner_json_data.get('email', None)
|
||||||
|
if owner_name and owner_email:
|
||||||
|
maintainers.append('%s <%s>' % (owner_name, owner_email))
|
||||||
|
|
||||||
maintainers = []
|
if layer.name == 'openembedded-core':
|
||||||
if readme_files:
|
layer.summary = 'Core metadata'
|
||||||
(desc, maintainers, deps) = readme_extract(readme_files[0])
|
layer.layer_type = 'A'
|
||||||
if desc:
|
elif layer.name == 'meta-oe':
|
||||||
layer.summary = layer.name
|
layer.summary = 'Additional shared OE metadata'
|
||||||
layer.description = desc
|
|
||||||
if maintainer_files:
|
|
||||||
maintainers.extend(maintainers_extract(readme_files[0]))
|
|
||||||
|
|
||||||
if (not maintainers) and 'github.com' in layer.vcs_url:
|
|
||||||
if json_data:
|
|
||||||
layer.summary = json_data['description']
|
|
||||||
layer.description = layer.summary
|
layer.description = layer.summary
|
||||||
if owner_json_data:
|
layer.layer_type = 'A'
|
||||||
owner_name = owner_json_data.get('name', None)
|
|
||||||
owner_email = owner_json_data.get('email', None)
|
|
||||||
if owner_name and owner_email:
|
|
||||||
maintainers.append('%s <%s>' % (owner_name, owner_email))
|
|
||||||
|
|
||||||
if layer.name == 'openembedded-core':
|
if maintainers:
|
||||||
layer.summary = 'Core metadata'
|
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
|
||||||
layer.layer_type = 'A'
|
for maintentry in maintainers:
|
||||||
elif layer.name == 'meta-oe':
|
res = maint_re.match(maintentry)
|
||||||
layer.summary = 'Additional shared OE metadata'
|
if res:
|
||||||
layer.description = layer.summary
|
maintainer = LayerMaintainer()
|
||||||
layer.layer_type = 'A'
|
maintainer.layerbranch = layerbranch
|
||||||
|
maintainer.name = res.group(1).strip()
|
||||||
|
maintainer.email = res.group(2)
|
||||||
|
if res.group(3):
|
||||||
|
maintainer.responsibility = res.group(3).strip()
|
||||||
|
maintainer.save()
|
||||||
|
|
||||||
if maintainers:
|
layer.save()
|
||||||
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
|
|
||||||
for maintentry in maintainers:
|
|
||||||
res = maint_re.match(maintentry)
|
|
||||||
if res:
|
|
||||||
maintainer = LayerMaintainer()
|
|
||||||
maintainer.layerbranch = layerbranch
|
|
||||||
maintainer.name = res.group(1).strip()
|
|
||||||
maintainer.email = res.group(2)
|
|
||||||
if res.group(3):
|
|
||||||
maintainer.responsibility = res.group(3).strip()
|
|
||||||
maintainer.save()
|
|
||||||
|
|
||||||
layer.save()
|
if options.dryrun:
|
||||||
|
raise DryRunRollbackException()
|
||||||
if options.dryrun:
|
except DryRunRollbackException:
|
||||||
transaction.rollback()
|
pass
|
||||||
else:
|
|
||||||
transaction.commit()
|
|
||||||
except:
|
|
||||||
transaction.rollback()
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
transaction.leave_transaction_management()
|
|
||||||
|
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user