mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-05 05:04:44 +02:00
79834a7144
This adds the SPDX-License-Identifier license headers to the majority of our source files to make it clearer exactly which license files are under. The bulk of the files are under GPL v2.0 with one found to be under V2.0 or later, some under MIT and some have dual license. There are some files which are potentially harder to classify where we've imported upstream code and those can be handled specifically in later commits. The COPYING file is replaced with LICENSE.X files which contain the full license texts. (Bitbake rev: ff237c33337f4da2ca06c3a2c49699bc26608a6b) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
401 lines
16 KiB
Python
401 lines
16 KiB
Python
# Copyright (C) 2016-2018 Wind River Systems, Inc.
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License version 2 as
|
|
# published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
# See the GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
|
|
import logging
|
|
import json
|
|
from urllib.parse import unquote
|
|
from urllib.parse import urlparse
|
|
|
|
import layerindexlib
|
|
import layerindexlib.plugin
|
|
|
|
logger = logging.getLogger('BitBake.layerindexlib.restapi')
|
|
|
|
def plugin_init(plugins):
|
|
return RestApiPlugin()
|
|
|
|
class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
|
|
def __init__(self):
|
|
self.type = "restapi"
|
|
|
|
def load_index(self, url, load):
|
|
"""
|
|
Fetches layer information from a local or remote layer index.
|
|
|
|
The return value is a LayerIndexObj.
|
|
|
|
url is the url to the rest api of the layer index, such as:
|
|
http://layers.openembedded.org/layerindex/api/
|
|
|
|
Or a local file...
|
|
"""
|
|
|
|
up = urlparse(url)
|
|
|
|
if up.scheme == 'file':
|
|
return self.load_index_file(up, url, load)
|
|
|
|
if up.scheme == 'http' or up.scheme == 'https':
|
|
return self.load_index_web(up, url, load)
|
|
|
|
raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
|
|
|
|
|
|
def load_index_file(self, up, url, load):
|
|
"""
|
|
Fetches layer information from a local file or directory.
|
|
|
|
The return value is a LayerIndexObj.
|
|
|
|
ud is the parsed url to the local file or directory.
|
|
"""
|
|
if not os.path.exists(up.path):
|
|
raise FileNotFoundError(up.path)
|
|
|
|
index = layerindexlib.LayerIndexObj()
|
|
|
|
index.config = {}
|
|
index.config['TYPE'] = self.type
|
|
index.config['URL'] = url
|
|
|
|
params = self.layerindex._parse_params(up.params)
|
|
|
|
if 'desc' in params:
|
|
index.config['DESCRIPTION'] = unquote(params['desc'])
|
|
else:
|
|
index.config['DESCRIPTION'] = up.path
|
|
|
|
if 'cache' in params:
|
|
index.config['CACHE'] = params['cache']
|
|
|
|
if 'branch' in params:
|
|
branches = params['branch'].split(',')
|
|
index.config['BRANCH'] = branches
|
|
else:
|
|
branches = ['*']
|
|
|
|
|
|
def load_cache(path, index, branches=[]):
|
|
logger.debug(1, 'Loading json file %s' % path)
|
|
with open(path, 'rt', encoding='utf-8') as f:
|
|
pindex = json.load(f)
|
|
|
|
# Filter the branches on loaded files...
|
|
newpBranch = []
|
|
for branch in branches:
|
|
if branch != '*':
|
|
if 'branches' in pindex:
|
|
for br in pindex['branches']:
|
|
if br['name'] == branch:
|
|
newpBranch.append(br)
|
|
else:
|
|
if 'branches' in pindex:
|
|
for br in pindex['branches']:
|
|
newpBranch.append(br)
|
|
|
|
if newpBranch:
|
|
index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
|
|
else:
|
|
logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
|
|
# No matching branches.. return nothing...
|
|
return
|
|
|
|
for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
|
|
("layerBranches", layerindexlib.LayerBranch),
|
|
("layerDependencies", layerindexlib.LayerDependency),
|
|
("recipes", layerindexlib.Recipe),
|
|
("machines", layerindexlib.Machine),
|
|
("distros", layerindexlib.Distro)]:
|
|
if lName in pindex:
|
|
index.add_raw_element(lName, lType, pindex[lName])
|
|
|
|
|
|
if not os.path.isdir(up.path):
|
|
load_cache(up.path, index, branches)
|
|
return index
|
|
|
|
logger.debug(1, 'Loading from dir %s...' % (up.path))
|
|
for (dirpath, _, filenames) in os.walk(up.path):
|
|
for filename in filenames:
|
|
if not filename.endswith('.json'):
|
|
continue
|
|
fpath = os.path.join(dirpath, filename)
|
|
load_cache(fpath, index, branches)
|
|
|
|
return index
|
|
|
|
|
|
def load_index_web(self, up, url, load):
|
|
"""
|
|
Fetches layer information from a remote layer index.
|
|
|
|
The return value is a LayerIndexObj.
|
|
|
|
ud is the parsed url to the rest api of the layer index, such as:
|
|
http://layers.openembedded.org/layerindex/api/
|
|
"""
|
|
|
|
def _get_json_response(apiurl=None, username=None, password=None, retry=True):
|
|
assert apiurl is not None
|
|
|
|
logger.debug(1, "fetching %s" % apiurl)
|
|
|
|
up = urlparse(apiurl)
|
|
|
|
username=up.username
|
|
password=up.password
|
|
|
|
# Strip username/password and params
|
|
if up.port:
|
|
up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
|
|
else:
|
|
up_stripped = up._replace(params="", netloc=up.hostname)
|
|
|
|
res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
|
|
|
|
try:
|
|
parsed = json.loads(res.read().decode('utf-8'))
|
|
except ConnectionResetError:
|
|
if retry:
|
|
logger.debug(1, "%s: Connection reset by peer. Retrying..." % url)
|
|
parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
|
|
logger.debug(1, "%s: retry successful.")
|
|
else:
|
|
raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
|
|
|
|
return parsed
|
|
|
|
index = layerindexlib.LayerIndexObj()
|
|
|
|
index.config = {}
|
|
index.config['TYPE'] = self.type
|
|
index.config['URL'] = url
|
|
|
|
params = self.layerindex._parse_params(up.params)
|
|
|
|
if 'desc' in params:
|
|
index.config['DESCRIPTION'] = unquote(params['desc'])
|
|
else:
|
|
index.config['DESCRIPTION'] = up.hostname
|
|
|
|
if 'cache' in params:
|
|
index.config['CACHE'] = params['cache']
|
|
|
|
if 'branch' in params:
|
|
branches = params['branch'].split(',')
|
|
index.config['BRANCH'] = branches
|
|
else:
|
|
branches = ['*']
|
|
|
|
try:
|
|
index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
|
|
except Exception as e:
|
|
raise layerindexlib.LayerIndexFetchError(url, e)
|
|
|
|
# Local raw index set...
|
|
pindex = {}
|
|
|
|
# Load all the requested branches at the same time time,
|
|
# a special branch of '*' means load all branches
|
|
filter = ""
|
|
if "*" not in branches:
|
|
filter = "?filter=name:%s" % "OR".join(branches)
|
|
|
|
logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
|
|
username=up.username, password=up.password)
|
|
if not pindex['branches']:
|
|
logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
|
|
return index
|
|
index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
|
|
|
|
# Load all of the layerItems (these can not be easily filtered)
|
|
logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
|
|
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
|
|
username=up.username, password=up.password)
|
|
if not pindex['layerItems']:
|
|
logger.debug(1, "No layers were found at url %s." % (url))
|
|
return index
|
|
index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
|
|
|
|
|
|
# From this point on load the contents for each branch. Otherwise we
|
|
# could run into a timeout.
|
|
for branch in index.branches:
|
|
filter = "?filter=branch__name:%s" % index.branches[branch].name
|
|
|
|
logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
|
|
username=up.username, password=up.password)
|
|
if not pindex['layerBranches']:
|
|
logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
|
|
return index
|
|
index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
|
|
|
|
|
|
# Load the rest, they all have a similar format
|
|
# Note: the layer index has a few more items, we can add them if necessary
|
|
# in the future.
|
|
filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
|
|
for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
|
|
("recipes", layerindexlib.Recipe),
|
|
("machines", layerindexlib.Machine),
|
|
("distros", layerindexlib.Distro)]:
|
|
if lName not in load:
|
|
continue
|
|
logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
|
|
|
|
# The link won't include username/password, so pull it from the original url
|
|
pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
|
|
username=up.username, password=up.password)
|
|
index.add_raw_element(lName, lType, pindex[lName])
|
|
|
|
return index
|
|
|
|
def store_index(self, url, index):
|
|
"""
|
|
Store layer information into a local file/dir.
|
|
|
|
The return value is a dictionary containing API,
|
|
layer, branch, dependency, recipe, machine, distro, information.
|
|
|
|
ud is a parsed url to a directory or file. If the path is a
|
|
directory, we will split the files into one file per layer.
|
|
If the path is to a file (exists or not) the entire DB will be
|
|
dumped into that one file.
|
|
"""
|
|
|
|
up = urlparse(url)
|
|
|
|
if up.scheme != 'file':
|
|
raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
|
|
|
|
logger.debug(1, "Storing to %s..." % up.path)
|
|
|
|
try:
|
|
layerbranches = index.layerBranches
|
|
except KeyError:
|
|
logger.error('No layerBranches to write.')
|
|
return
|
|
|
|
|
|
def filter_item(layerbranchid, objects):
|
|
filtered = []
|
|
for obj in getattr(index, objects, None):
|
|
try:
|
|
if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
|
|
filtered.append(getattr(index, objects)[obj]._data)
|
|
except AttributeError:
|
|
logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
|
|
# No simple filter method, just include it...
|
|
try:
|
|
filtered.append(getattr(index, objects)[obj]._data)
|
|
except AttributeError:
|
|
logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
|
|
filtered.append(obj)
|
|
return filtered
|
|
|
|
|
|
# Write out to a single file.
|
|
# Filter out unnecessary items, then sort as we write for determinism
|
|
if not os.path.isdir(up.path):
|
|
pindex = {}
|
|
|
|
pindex['branches'] = []
|
|
pindex['layerItems'] = []
|
|
pindex['layerBranches'] = []
|
|
|
|
for layerbranchid in layerbranches:
|
|
if layerbranches[layerbranchid].branch._data not in pindex['branches']:
|
|
pindex['branches'].append(layerbranches[layerbranchid].branch._data)
|
|
|
|
if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
|
|
pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
|
|
|
|
if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
|
|
pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
|
|
|
|
for entry in index._index:
|
|
# Skip local items, apilinks and items already processed
|
|
if entry in index.config['local'] or \
|
|
entry == 'apilinks' or \
|
|
entry == 'branches' or \
|
|
entry == 'layerBranches' or \
|
|
entry == 'layerItems':
|
|
continue
|
|
if entry not in pindex:
|
|
pindex[entry] = []
|
|
pindex[entry].extend(filter_item(layerbranchid, entry))
|
|
|
|
bb.debug(1, 'Writing index to %s' % up.path)
|
|
with open(up.path, 'wt') as f:
|
|
json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
|
|
return
|
|
|
|
|
|
# Write out to a directory one file per layerBranch
|
|
# Prepare all layer related items, to create a minimal file.
|
|
# We have to sort the entries as we write so they are deterministic
|
|
for layerbranchid in layerbranches:
|
|
pindex = {}
|
|
|
|
for entry in index._index:
|
|
# Skip local items, apilinks and items already processed
|
|
if entry in index.config['local'] or \
|
|
entry == 'apilinks' or \
|
|
entry == 'branches' or \
|
|
entry == 'layerBranches' or \
|
|
entry == 'layerItems':
|
|
continue
|
|
pindex[entry] = filter_item(layerbranchid, entry)
|
|
|
|
# Add the layer we're processing as the first one...
|
|
pindex['branches'] = [layerbranches[layerbranchid].branch._data]
|
|
pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
|
|
pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
|
|
|
|
# We also need to include the layerbranch for any dependencies...
|
|
for layerdep in pindex['layerDependencies']:
|
|
layerdependency = layerindexlib.LayerDependency(index, layerdep)
|
|
|
|
layeritem = layerdependency.dependency
|
|
layerbranch = layerdependency.dependency_layerBranch
|
|
|
|
# We need to avoid duplicates...
|
|
if layeritem._data not in pindex['layerItems']:
|
|
pindex['layerItems'].append(layeritem._data)
|
|
|
|
if layerbranch._data not in pindex['layerBranches']:
|
|
pindex['layerBranches'].append(layerbranch._data)
|
|
|
|
# apply mirroring adjustments here....
|
|
|
|
fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
|
|
fname = fname.translate(str.maketrans('/ ', '__'))
|
|
fpath = os.path.join(up.path, fname)
|
|
|
|
bb.debug(1, 'Writing index to %s' % fpath + '.json')
|
|
with open(fpath + '.json', 'wt') as f:
|
|
json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
|