MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-06 13:44:47 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

import_layers: use recipesExtended viewset

Browse Source 
The new viewset has a different URL and uses pagination, so we need to
take that into account.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2019-09-16 16:33:58 +12:00
parent ee1ff214a0
commit cb4bd5e45b
1 changed files with 54 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
layerindex/tools/import_layers.py
View File

@ -89,7 +89,7 @@ def main():
layerbranches_url = jsdata['layerBranches'] layerbranches_url = jsdata['layerBranches']
layermaintainers_url = jsdata.get('layerMaintainers', None) layermaintainers_url = jsdata.get('layerMaintainers', None)
layernotes_url = jsdata.get('layerNotes', None) layernotes_url = jsdata.get('layerNotes', None)
recipes_url = jsdata.get('recipes', None) recipes_url = jsdata.get('recipesExtended', None)
machines_url = jsdata.get('machines', None) machines_url = jsdata.get('machines', None)
distros_url = jsdata.get('distros', None) distros_url = jsdata.get('distros', None)
classes_url = jsdata.get('classes', None) classes_url = jsdata.get('classes', None)
@ -186,20 +186,25 @@ def main():
# The parent field always needs to be part of the keys # The parent field always needs to be part of the keys
keys = key_fields + [parentfield] keys = key_fields + [parentfield]
def fetch_api_url(api_url):
rq = urllib.request.Request(api_url)
data = urllib.request.urlopen(rq).read()
return json.loads(data.decode('utf-8'))
if url: if url:
if parent_orig_id is None: if parent_orig_id is None:
raise Exception('import_child_items: if url is specified then parent_orig_id must also be specified') raise Exception('import_child_items: if url is specified then parent_orig_id must also be specified')
rq = urllib.request.Request(url + '?filter=%s:%s' % (parentfield, parent_orig_id)) childjsdata = fetch_api_url(url + '?filter=%s:%s' % (parentfield, parent_orig_id))
data = urllib.request.urlopen(rq).read()
childjslist = json.loads(data.decode('utf-8'))
elif childlist is not None: elif childlist is not None:
childjslist = childlist childjsdata = childlist
else: else:
raise Exception('import_child_items: either url or childlist must be specified') raise Exception('import_child_items: either url or childlist must be specified')
manager = getattr(parentobj, objclass.__name__.lower() + '_set') manager = getattr(parentobj, objclass.__name__.lower() + '_set')
existing_ids = list(manager.values_list('id', flat=True)) existing_ids = list(manager.values_list('id', flat=True))
updated_ids = [] updated_ids = []
def import_list(childjslist):
for childjs in childjslist: for childjs in childjslist:
vals = {} vals = {}
for key, value in childjs.items(): for key, value in childjs.items():
@ -235,6 +240,19 @@ def main():
if not created: if not created:
if obj.id in existing_ids: if obj.id in existing_ids:
existing_ids.remove(obj.id) existing_ids.remove(obj.id)
if 'results' in childjsdata:
while True:
import_list(childjsdata['results'])
if childjsdata.get('next', None):
childjsdata = fetch_api_url(childjsdata['next'])
if not 'results' in childjsdata:
break
else:
break
else:
import_list(childjsdata)
for idv in existing_ids: for idv in existing_ids:
objclass.objects.filter(id=idv).delete() objclass.objects.filter(id=idv).delete()