MIRRORS/layerindex-web
1
0
Fork 0
mirror of git://git.yoctoproject.org/layerindex-web.git synced 2025-07-19 20:59:01 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

import_layers: enable incremental LayerItem/LayerBranch import

Browse Source 
Don't assume we need to create these objects - check the updated field
value against what the API reports and update the existing object if the
latter is newer.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2019-08-05 11:10:00 +12:00
parent 8861ccfa19
commit 5352a227ab
1 changed files with 32 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
layerindex/tools/import_layers.py
View File

@ -17,6 +17,8 @@ import logging
import subprocess import subprocess
import urllib.request import urllib.request
import json import json
import datetime
from django.utils import timezone
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
@ -29,6 +31,14 @@ class DryRunRollbackException(Exception):
logger = utils.logger_create('LayerIndexImport') logger = utils.logger_create('LayerIndexImport')
iso8601_date_re = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}')
def datetime_hook(jsdict):
for key, value in jsdict.items():
if isinstance(value, str) and iso8601_date_re.match(value):
jsdict[key] = timezone.make_naive(datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z"))
return jsdict
def main(): def main():
valid_layer_name = re.compile('[-\w]+$') valid_layer_name = re.compile('[-\w]+$')
@ -105,19 +115,23 @@ def main():
logger.debug('Importing layers') logger.debug('Importing layers')
rq = urllib.request.Request(layers_url) rq = urllib.request.Request(layers_url)
data = urllib.request.urlopen(rq).read() data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8')) jsdata = json.loads(data.decode('utf-8'), object_hook=datetime_hook)
layer_idmap = {} layer_idmap = {}
exclude_fields = ['id', 'updated'] exclude_fields = ['id', 'updated']
for layerjs in jsdata: for layerjs in jsdata:
res = LayerItem.objects.filter(name=layerjs['name']) layeritem = LayerItem.objects.filter(name=layerjs['name']).first()
if res: if layeritem:
# Already have this layer # Already have this layer
logger.debug('Skipping layer %s, already in database' % layerjs['name']) if layerjs['updated'] <= layeritem.updated:
layer_idmap[layerjs['id']] = res[0] logger.debug('Skipping layer %s, already up-to-date' % layerjs['name'])
continue layer_idmap[layerjs['id']] = layeritem
logger.debug('Adding layer %s' % layerjs['name']) continue
layeritem = LayerItem() else:
logger.debug('Updating layer %s' % layerjs['name'])
else:
logger.debug('Adding layer %s' % layerjs['name'])
layeritem = LayerItem()
for key, value in layerjs.items(): for key, value in layerjs.items():
if key in exclude_fields: if key in exclude_fields:
continue continue
@ -144,21 +158,24 @@ def main():
# We didn't import this layer, skip it # We didn't import this layer, skip it
logger.debug('Skipping layerbranch %s, layer not imported' % layerbranchjs['id']) logger.debug('Skipping layerbranch %s, layer not imported' % layerbranchjs['id'])
continue continue
res = LayerBranch.objects.filter(layer=layer).filter(branch=branch) layerbranch = LayerBranch.objects.filter(layer=layer).filter(branch=branch).first()
if res: if layerbranch:
# The layerbranch already exists (this will occur for layers # The layerbranch already exists (this will occur for layers
# that already existed, since we need to have those in layer_idmap # that already existed, since we need to have those in layer_idmap
# to be able to import layer dependencies) # to be able to import layer dependencies)
logger.debug('Skipping layerbranch %s, already exists' % layerbranchjs['id']) if layerbranchjs['updated'] <= layerbranch.updated:
continue logger.debug('Skipping layerbranch %s, already up-to-date' % layerbranchjs['id'])
layerbranch_idmap[layerbranchjs['id']] = layerbranch
continue
else:
layerbranch = LayerBranch()
layerbranch.branch = branch
layerbranch.layer = layer
layerbranch = LayerBranch()
for key, value in layerbranchjs.items(): for key, value in layerbranchjs.items():
if key in exclude_fields: if key in exclude_fields:
continue continue
setattr(layerbranch, key, value) setattr(layerbranch, key, value)
layerbranch.branch = branch
layerbranch.layer = layer
layerbranch.save() layerbranch.save()
layerbranch_idmap[layerbranchjs['id']] = layerbranch layerbranch_idmap[layerbranchjs['id']] = layerbranch