mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
75f87db413
The bitbake logger overrode the definition of the debug() logging call to include a debug level, but this causes problems with code that may be using standard python logging, since the extra argument is interpreted differently. Instead, change the bitbake loggers debug() call to match the python logger call and add a debug2() and debug3() API to replace calls that were logging to a different debug level. [RP: Small fix to ensure bb.debug calls bbdebug()] (Bitbake rev: f68682a79d83e6399eb403f30a1f113516575f51) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
393 lines
15 KiB
Python
393 lines
15 KiB
Python
# Copyright (C) 2016-2018 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import logging
|
|
import json
|
|
import os
|
|
|
|
from urllib.parse import unquote
|
|
from urllib.parse import urlparse
|
|
|
|
import bb
|
|
|
|
import layerindexlib
|
|
import layerindexlib.plugin
|
|
|
|
logger = logging.getLogger('BitBake.layerindexlib.restapi')
|
|
|
|
def plugin_init(plugins):
|
|
return RestApiPlugin()
|
|
|
|
class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
|
|
def __init__(self):
|
|
self.type = "restapi"
|
|
|
|
def load_index(self, url, load):
|
|
"""
|
|
Fetches layer information from a local or remote layer index.
|
|
|
|
The return value is a LayerIndexObj.
|
|
|
|
url is the url to the rest api of the layer index, such as:
|
|
http://layers.openembedded.org/layerindex/api/
|
|
|
|
Or a local file...
|
|
"""
|
|
|
|
up = urlparse(url)
|
|
|
|
if up.scheme == 'file':
|
|
return self.load_index_file(up, url, load)
|
|
|
|
if up.scheme == 'http' or up.scheme == 'https':
|
|
return self.load_index_web(up, url, load)
|
|
|
|
raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
|
|
|
|
|
|
def load_index_file(self, up, url, load):
|
|
"""
|
|
Fetches layer information from a local file or directory.
|
|
|
|
The return value is a LayerIndexObj.
|
|
|
|
ud is the parsed url to the local file or directory.
|
|
"""
|
|
if not os.path.exists(up.path):
|
|
raise FileNotFoundError(up.path)
|
|
|
|
index = layerindexlib.LayerIndexObj()
|
|
|
|
index.config = {}
|
|
index.config['TYPE'] = self.type
|
|
index.config['URL'] = url
|
|
|
|
params = self.layerindex._parse_params(up.params)
|
|
|
|
if 'desc' in params:
|
|
index.config['DESCRIPTION'] = unquote(params['desc'])
|
|
else:
|
|
index.config['DESCRIPTION'] = up.path
|
|
|
|
if 'cache' in params:
|
|
index.config['CACHE'] = params['cache']
|
|
|
|
if 'branch' in params:
|
|
branches = params['branch'].split(',')
|
|
index.config['BRANCH'] = branches
|
|
else:
|
|
branches = ['*']
|
|
|
|
|
|
def load_cache(path, index, branches=[]):
|
|
logger.debug('Loading json file %s' % path)
|
|
with open(path, 'rt', encoding='utf-8') as f:
|
|
pindex = json.load(f)
|
|
|
|
# Filter the branches on loaded files...
|
|
newpBranch = []
|
|
for branch in branches:
|
|
if branch != '*':
|
|
if 'branches' in pindex:
|
|
for br in pindex['branches']:
|
|
if br['name'] == branch:
|
|
newpBranch.append(br)
|
|
else:
|
|
if 'branches' in pindex:
|
|
for br in pindex['branches']:
|
|
newpBranch.append(br)
|
|
|
|
if newpBranch:
|
|
index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
|
|
else:
|
|
logger.debug('No matching branches (%s) in index file(s)' % branches)
|
|
# No matching branches.. return nothing...
|
|
return
|
|
|
|
for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
|
|
("layerBranches", layerindexlib.LayerBranch),
|
|
("layerDependencies", layerindexlib.LayerDependency),
|
|
("recipes", layerindexlib.Recipe),
|
|
("machines", layerindexlib.Machine),
|
|
("distros", layerindexlib.Distro)]:
|
|
if lName in pindex:
|
|
index.add_raw_element(lName, lType, pindex[lName])
|
|
|
|
|
|
if not os.path.isdir(up.path):
|
|
load_cache(up.path, index, branches)
|
|
return index
|
|
|
|
logger.debug('Loading from dir %s...' % (up.path))
|
|
for (dirpath, _, filenames) in os.walk(up.path):
|
|
for filename in filenames:
|
|
if not filename.endswith('.json'):
|
|
continue
|
|
fpath = os.path.join(dirpath, filename)
|
|
load_cache(fpath, index, branches)
|
|
|
|
return index
|
|
|
|
|
|
def load_index_web(self, up, url, load):
|
|
"""
|
|
Fetches layer information from a remote layer index.
|
|
|
|
The return value is a LayerIndexObj.
|
|
|
|
ud is the parsed url to the rest api of the layer index, such as:
|
|
http://layers.openembedded.org/layerindex/api/
|
|
"""
|
|
|
|
def _get_json_response(apiurl=None, username=None, password=None, retry=True):
|
|
assert apiurl is not None
|
|
|
|
logger.debug("fetching %s" % apiurl)
|
|
|
|
up = urlparse(apiurl)
|
|
|
|
username=up.username
|
|
password=up.password
|
|
|
|
# Strip username/password and params
|
|
if up.port:
|
|
up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
|
|
else:
|
|
up_stripped = up._replace(params="", netloc=up.hostname)
|
|
|
|
res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
|
|
|
|
try:
|
|
parsed = json.loads(res.read().decode('utf-8'))
|
|
except ConnectionResetError:
|
|
if retry:
|
|
logger.debug("%s: Connection reset by peer. Retrying..." % url)
|
|
parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
|
|
logger.debug("%s: retry successful.")
|
|
else:
|
|
raise layerindexlib.LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
|
|
|
|
return parsed
|
|
|
|
index = layerindexlib.LayerIndexObj()
|
|
|
|
index.config = {}
|
|
index.config['TYPE'] = self.type
|
|
index.config['URL'] = url
|
|
|
|
params = self.layerindex._parse_params(up.params)
|
|
|
|
if 'desc' in params:
|
|
index.config['DESCRIPTION'] = unquote(params['desc'])
|
|
else:
|
|
index.config['DESCRIPTION'] = up.hostname
|
|
|
|
if 'cache' in params:
|
|
index.config['CACHE'] = params['cache']
|
|
|
|
if 'branch' in params:
|
|
branches = params['branch'].split(',')
|
|
index.config['BRANCH'] = branches
|
|
else:
|
|
branches = ['*']
|
|
|
|
try:
|
|
index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
|
|
except Exception as e:
|
|
raise layerindexlib.LayerIndexFetchError(url, e)
|
|
|
|
# Local raw index set...
|
|
pindex = {}
|
|
|
|
# Load all the requested branches at the same time time,
|
|
# a special branch of '*' means load all branches
|
|
filter = ""
|
|
if "*" not in branches:
|
|
filter = "?filter=name:%s" % "OR".join(branches)
|
|
|
|
logger.debug("Loading %s from %s" % (branches, index.apilinks['branches']))
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
|
|
username=up.username, password=up.password)
|
|
if not pindex['branches']:
|
|
logger.debug("No valid branches (%s) found at url %s." % (branch, url))
|
|
return index
|
|
index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
|
|
|
|
# Load all of the layerItems (these can not be easily filtered)
|
|
logger.debug("Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
|
|
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
|
|
username=up.username, password=up.password)
|
|
if not pindex['layerItems']:
|
|
logger.debug("No layers were found at url %s." % (url))
|
|
return index
|
|
index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
|
|
|
|
|
|
# From this point on load the contents for each branch. Otherwise we
|
|
# could run into a timeout.
|
|
for branch in index.branches:
|
|
filter = "?filter=branch__name:%s" % index.branches[branch].name
|
|
|
|
logger.debug("Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
|
|
username=up.username, password=up.password)
|
|
if not pindex['layerBranches']:
|
|
logger.debug("No valid layer branches (%s) found at url %s." % (branches or "*", url))
|
|
return index
|
|
index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
|
|
|
|
|
|
# Load the rest, they all have a similar format
|
|
# Note: the layer index has a few more items, we can add them if necessary
|
|
# in the future.
|
|
filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
|
|
for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
|
|
("recipes", layerindexlib.Recipe),
|
|
("machines", layerindexlib.Machine),
|
|
("distros", layerindexlib.Distro)]:
|
|
if lName not in load:
|
|
continue
|
|
logger.debug("Loading %s from %s" % (lName, index.apilinks[lName]))
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
|
|
username=up.username, password=up.password)
|
|
index.add_raw_element(lName, lType, pindex[lName])
|
|
|
|
return index
|
|
|
|
def store_index(self, url, index):
|
|
"""
|
|
Store layer information into a local file/dir.
|
|
|
|
The return value is a dictionary containing API,
|
|
layer, branch, dependency, recipe, machine, distro, information.
|
|
|
|
ud is a parsed url to a directory or file. If the path is a
|
|
directory, we will split the files into one file per layer.
|
|
If the path is to a file (exists or not) the entire DB will be
|
|
dumped into that one file.
|
|
"""
|
|
|
|
up = urlparse(url)
|
|
|
|
if up.scheme != 'file':
|
|
raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
|
|
|
|
logger.debug("Storing to %s..." % up.path)
|
|
|
|
try:
|
|
layerbranches = index.layerBranches
|
|
except KeyError:
|
|
logger.error('No layerBranches to write.')
|
|
return
|
|
|
|
|
|
def filter_item(layerbranchid, objects):
|
|
filtered = []
|
|
for obj in getattr(index, objects, None):
|
|
try:
|
|
if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
|
|
filtered.append(getattr(index, objects)[obj]._data)
|
|
except AttributeError:
|
|
logger.debug('No obj.layerbranch_id: %s' % objects)
|
|
# No simple filter method, just include it...
|
|
try:
|
|
filtered.append(getattr(index, objects)[obj]._data)
|
|
except AttributeError:
|
|
logger.debug('No obj._data: %s %s' % (objects, type(obj)))
|
|
filtered.append(obj)
|
|
return filtered
|
|
|
|
|
|
# Write out to a single file.
|
|
# Filter out unnecessary items, then sort as we write for determinism
|
|
if not os.path.isdir(up.path):
|
|
pindex = {}
|
|
|
|
pindex['branches'] = []
|
|
pindex['layerItems'] = []
|
|
pindex['layerBranches'] = []
|
|
|
|
for layerbranchid in layerbranches:
|
|
if layerbranches[layerbranchid].branch._data not in pindex['branches']:
|
|
pindex['branches'].append(layerbranches[layerbranchid].branch._data)
|
|
|
|
if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
|
|
pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
|
|
|
|
if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
|
|
pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
|
|
|
|
for entry in index._index:
|
|
# Skip local items, apilinks and items already processed
|
|
if entry in index.config['local'] or \
|
|
entry == 'apilinks' or \
|
|
entry == 'branches' or \
|
|
entry == 'layerBranches' or \
|
|
entry == 'layerItems':
|
|
continue
|
|
if entry not in pindex:
|
|
pindex[entry] = []
|
|
pindex[entry].extend(filter_item(layerbranchid, entry))
|
|
|
|
bb.debug(1, 'Writing index to %s' % up.path)
|
|
with open(up.path, 'wt') as f:
|
|
json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
|
|
return
|
|
|
|
|
|
# Write out to a directory one file per layerBranch
|
|
# Prepare all layer related items, to create a minimal file.
|
|
# We have to sort the entries as we write so they are deterministic
|
|
for layerbranchid in layerbranches:
|
|
pindex = {}
|
|
|
|
for entry in index._index:
|
|
# Skip local items, apilinks and items already processed
|
|
if entry in index.config['local'] or \
|
|
entry == 'apilinks' or \
|
|
entry == 'branches' or \
|
|
entry == 'layerBranches' or \
|
|
entry == 'layerItems':
|
|
continue
|
|
pindex[entry] = filter_item(layerbranchid, entry)
|
|
|
|
# Add the layer we're processing as the first one...
|
|
pindex['branches'] = [layerbranches[layerbranchid].branch._data]
|
|
pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
|
|
pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
|
|
|
|
# We also need to include the layerbranch for any dependencies...
|
|
for layerdep in pindex['layerDependencies']:
|
|
layerdependency = layerindexlib.LayerDependency(index, layerdep)
|
|
|
|
layeritem = layerdependency.dependency
|
|
layerbranch = layerdependency.dependency_layerBranch
|
|
|
|
# We need to avoid duplicates...
|
|
if layeritem._data not in pindex['layerItems']:
|
|
pindex['layerItems'].append(layeritem._data)
|
|
|
|
if layerbranch._data not in pindex['layerBranches']:
|
|
pindex['layerBranches'].append(layerbranch._data)
|
|
|
|
# apply mirroring adjustments here....
|
|
|
|
fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
|
|
fname = fname.translate(str.maketrans('/ ', '__'))
|
|
fpath = os.path.join(up.path, fname)
|
|
|
|
bb.debug(1, 'Writing index to %s' % fpath + '.json')
|
|
with open(fpath + '.json', 'wt') as f:
|
|
json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
|