Add recipe bulk change feature

This provides a way to set "meta" fields (SUMMARY, DESCRIPTION,
HOMEPAGE, BUGTRACKER, SECTION, and LICENSE) for a number of recipes at
once, and then download those changes in the form of one or more patch
files which can be submitted for merging into the layer.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
This commit is contained in:
Paul Eggleton 2013-03-20 17:09:27 +00:00
parent 1a9f73d4a7
commit 84709dbca6
16 changed files with 1102 additions and 12 deletions

1
TODO
View File

@ -21,7 +21,6 @@ Later:
* Cancel button on edit form?
* Query backend service? i.e. special URL to query information for external apps/scripts
* Add comparison to duplicates page
* Tool for editing SUMMARY/DESCRIPTION? [Paul working on this]
* Dynamic loading/filtering for recipes list
* Some way to notify the user when they search for something that has been renamed / replaced / deprecated?
* Create simple script to check for unlisted layer subdirectories in all repos

View File

@ -80,6 +80,15 @@ class BBClassAdmin(admin.ModelAdmin):
def has_delete_permission(self, request, obj=None):
return False
class RecipeChangeInline(admin.StackedInline):
model = RecipeChange
class RecipeChangesetAdmin(admin.ModelAdmin):
model = RecipeChangeset
inlines = [
RecipeChangeInline
]
admin.site.register(Branch, BranchAdmin)
admin.site.register(LayerItem, LayerItemAdmin)
admin.site.register(LayerBranch, LayerBranchAdmin)
@ -91,3 +100,4 @@ admin.site.register(RecipeFileDependency)
admin.site.register(Machine, MachineAdmin)
admin.site.register(BBAppend, BBAppendAdmin)
admin.site.register(BBClass, BBClassAdmin)
admin.site.register(RecipeChangeset, RecipeChangesetAdmin)

237
layerindex/bulkchange.py Normal file
View File

@ -0,0 +1,237 @@
#!/usr/bin/env python
# layerindex-web - bulk change implementation
#
# Copyright (C) 2013 Intel Corporation
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
import os
import os.path
import tempfile
import tarfile
import textwrap
import difflib
import recipeparse
import utils
import shutil
from django.utils.datastructures import SortedDict
logger = utils.logger_create('LayerIndexImport')
# Help us to find places to insert values
recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRC_URI', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
# Variables that sometimes are a bit long but shouldn't be wrapped
nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER', 'LIC_FILES_CHKSUM']
meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']
def generate_patches(tinfoil, fetchdir, changeset, outputdir):
tmpoutdir = tempfile.mkdtemp(dir=outputdir)
last_layer = None
patchname = ''
patches = []
outfile = None
try:
for change in changeset.recipechange_set.all().order_by('recipe__layerbranch'):
fields = change.changed_fields(mapped=True)
if fields:
layerbranch = change.recipe.layerbranch
layer = layerbranch.layer
if last_layer != layer:
patchname = "%s.patch" % layer.name
patches.append(patchname)
layerfetchdir = os.path.join(fetchdir, layer.get_fetch_dir())
recipeparse.checkout_layer_branch(layerbranch, layerfetchdir)
layerdir = os.path.join(layerfetchdir, layerbranch.vcs_subdir)
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
if outfile:
outfile.close()
outfile = open(os.path.join(tmpoutdir, patchname), 'w')
last_layer = layer
recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
varlist = list(set(fields.keys() + meta_vars))
varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
filevars = localise_file_vars(recipefile, varfiles, fields.keys())
for f, fvars in filevars.items():
filefields = dict((k, fields[k]) for k in fvars)
patch = patch_recipe(f, layerfetchdir, filefields)
for line in patch:
outfile.write(line)
finally:
if outfile:
outfile.close()
# If we have more than one patch, tar it up, otherwise just return the single patch file
ret = None
if len(patches) > 1:
(tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir)
tmptarfile = os.fdopen(tmptarfd, "w")
tar = tarfile.open(None, "w:gz", tmptarfile)
for patch in patches:
patchfn = os.path.join(tmpoutdir, patch)
tar.add(patchfn)
tar.close()
ret = tmptarname
elif len(patches) == 1:
(tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir)
tmppatchfile = os.fdopen(tmppatchfd, "w")
with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile:
shutil.copyfileobj(patchfile, tmppatchfile)
tmppatchfile.close()
ret = tmppatchname
shutil.rmtree(tmpoutdir)
return ret
def patch_recipe(fn, relpath, values):
"""Update or insert variable values into a recipe file.
Note that some manual inspection/intervention may be required
since this cannot handle all situations.
"""
remainingnames = {}
for k in values.keys():
remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
remainingnames = SortedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))
with tempfile.NamedTemporaryFile('w', delete=False) as tf:
def outputvalue(name):
rawtext = '%s = "%s"\n' % (name, values[name])
if name in nowrap_vars:
tf.write(rawtext)
else:
wrapped = textwrap.wrap(rawtext)
for wrapline in wrapped[:-1]:
tf.write('%s \\\n' % wrapline)
tf.write('%s\n' % wrapped[-1])
tfn = tf.name
with open(fn, 'r') as f:
# First runthrough - find existing names (so we know not to insert based on recipe_progression)
# Second runthrough - make the changes
existingnames = []
for runthrough in [1, 2]:
currname = None
for line in f:
if not currname:
insert = False
for k in remainingnames.keys():
for p in recipe_progression:
if line.startswith(p):
if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
outputvalue(k)
del remainingnames[k]
break
for k in remainingnames.keys():
if line.startswith(k):
currname = k
if runthrough == 1:
existingnames.append(k)
else:
del remainingnames[k]
break
if currname and runthrough > 1:
outputvalue(currname)
if currname:
sline = line.rstrip()
if not sline.endswith('\\'):
currname = None
continue
if runthrough > 1:
tf.write(line)
f.seek(0)
if remainingnames:
tf.write('\n')
for k in remainingnames.keys():
outputvalue(k)
fromlines = open(fn, 'U').readlines()
tolines = open(tfn, 'U').readlines()
relfn = os.path.relpath(fn, relpath)
diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
os.remove(tfn)
return diff
def localise_file_vars(fn, varfiles, varlist):
from collections import defaultdict
fndir = os.path.dirname(fn) + os.sep
first_meta_file = None
for v in meta_vars:
f = varfiles.get(v, None)
if f:
actualdir = os.path.dirname(f) + os.sep
if actualdir.startswith(fndir):
first_meta_file = f
break
filevars = defaultdict(list)
for v in varlist:
f = varfiles[v]
# Only return files that are in the same directory as the recipe or in some directory below there
# (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
# in if we were going to set a value specific to this recipe)
if f:
actualfile = f
else:
# Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
if first_meta_file:
actualfile = first_meta_file
else:
actualfile = fn
actualdir = os.path.dirname(actualfile) + os.sep
if not actualdir.startswith(fndir):
actualfile = fn
filevars[actualfile].append(v)
return filevars
def get_changeset(pk):
from layerindex.models import RecipeChangeset
res = list(RecipeChangeset.objects.filter(pk=pk)[:1])
if res:
return res[0]
return None
def usage():
print("Usage: bulkchange.py <id> <outputdir>")
def main():
if '--help' in sys.argv:
usage()
sys.exit(0)
if len(sys.argv) < 3:
usage()
sys.exit(1)
utils.setup_django()
import settings
branch = utils.get_branch('master')
fetchdir = settings.LAYER_FETCH_DIR
bitbakepath = os.path.join(fetchdir, 'bitbake')
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, True)
changeset = get_changeset(sys.argv[1])
if not changeset:
sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1])
sys.exit(1)
outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
if outp:
print outp
else:
sys.stderr.write("No changes to write\n")
sys.exit(1)
shutil.rmtree(tempdir)
sys.exit(0)
if __name__ == "__main__":
main()

View File

@ -4,10 +4,10 @@
#
# Licensed under the MIT license, see COPYING.MIT for details
from layerindex.models import LayerItem, LayerBranch, LayerMaintainer, LayerNote
from layerindex.models import LayerItem, LayerBranch, LayerMaintainer, LayerNote, RecipeChangeset, RecipeChange
from django import forms
from django.core.validators import URLValidator, RegexValidator, email_re
from django.forms.models import inlineformset_factory
from django.forms.models import inlineformset_factory, modelformset_factory
from captcha.fields import CaptchaField
from django.contrib.auth.models import User
import re
@ -147,3 +147,58 @@ class EditProfileForm(forms.ModelForm):
class Meta:
model = User
fields = ('first_name', 'last_name', 'email')
class AdvancedRecipeSearchForm(forms.Form):
FIELD_CHOICES = (
('pn', 'Name'),
('summary', 'Summary'),
('description', 'Description'),
('homepage', 'Homepage'),
('bugtracker', 'Bug tracker'),
('section', 'Section'),
('license', 'License'),
)
MATCH_TYPE_CHOICES = (
('C', 'contains'),
('N', 'does not contain'),
('E', 'equals'),
('B', 'is blank'),
)
field = forms.ChoiceField(choices=FIELD_CHOICES)
match_type = forms.ChoiceField(choices=MATCH_TYPE_CHOICES)
value = forms.CharField(max_length=255, required=False)
layer = forms.ModelChoiceField(queryset=LayerItem.objects.filter(status='P').order_by('name'), empty_label="(any)", required=False)
class RecipeChangesetForm(forms.ModelForm):
class Meta:
model = RecipeChangeset
fields = ('name',)
class BulkChangeEditForm(forms.ModelForm):
class Meta:
model = RecipeChange
fields = ('summary', 'description', 'homepage', 'bugtracker', 'section', 'license')
def __init__(self, *args, **kwargs):
instance = kwargs.get('instance', None)
initial = kwargs.get('initial', {})
if instance:
recipe = instance.recipe
if recipe:
for fieldname in self._meta.fields:
if not getattr(instance, fieldname):
initial[fieldname] = getattr(recipe, fieldname)
kwargs['initial'] = initial
super(BulkChangeEditForm, self).__init__(*args, **kwargs)
def clear_same_values(self):
for fieldname in self._meta.fields:
oldval = getattr(self.instance.recipe, fieldname)
newval = getattr(self.instance, fieldname)
if oldval == newval:
setattr(self.instance, fieldname, '')
BulkChangeEditFormSet = modelformset_factory(RecipeChange, form=BulkChangeEditForm, extra=0)

View File

@ -0,0 +1,199 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding model 'RecipeChange'
db.create_table('layerindex_recipechange', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('changeset', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['layerindex.RecipeChangeset'])),
('recipe', self.gf('django.db.models.fields.related.ForeignKey')(related_name='+', to=orm['layerindex.Recipe'])),
('summary', self.gf('django.db.models.fields.CharField')(max_length=100, blank=True)),
('description', self.gf('django.db.models.fields.TextField')(blank=True)),
('section', self.gf('django.db.models.fields.CharField')(max_length=100, blank=True)),
('license', self.gf('django.db.models.fields.CharField')(max_length=100, blank=True)),
('homepage', self.gf('django.db.models.fields.URLField')(max_length=200, blank=True)),
('bugtracker', self.gf('django.db.models.fields.URLField')(max_length=200, blank=True)),
))
db.send_create_signal('layerindex', ['RecipeChange'])
# Adding model 'RecipeChangeset'
db.create_table('layerindex_recipechangeset', (
('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])),
('name', self.gf('django.db.models.fields.CharField')(max_length=255)),
))
db.send_create_signal('layerindex', ['RecipeChangeset'])
def backwards(self, orm):
# Deleting model 'RecipeChange'
db.delete_table('layerindex_recipechange')
# Deleting model 'RecipeChangeset'
db.delete_table('layerindex_recipechangeset')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'layerindex.bbappend': {
'Meta': {'object_name': 'BBAppend'},
'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
},
'layerindex.bbclass': {
'Meta': {'object_name': 'BBClass'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'layerindex.branch': {
'Meta': {'object_name': 'Branch'},
'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
'sort_priority': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
},
'layerindex.layerbranch': {
'Meta': {'object_name': 'LayerBranch'},
'actual_branch': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
'vcs_last_rev': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
'vcs_subdir': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'})
},
'layerindex.layerdependency': {
'Meta': {'object_name': 'LayerDependency'},
'dependency': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependents_set'", 'to': "orm['layerindex.LayerItem']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependencies_set'", 'to': "orm['layerindex.LayerBranch']"})
},
'layerindex.layeritem': {
'Meta': {'object_name': 'LayerItem'},
'description': ('django.db.models.fields.TextField', [], {}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
'mailing_list_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
'usage_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'vcs_web_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
},
'layerindex.layermaintainer': {
'Meta': {'object_name': 'LayerMaintainer'},
'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'responsibility': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
},
'layerindex.layernote': {
'Meta': {'object_name': 'LayerNote'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
'text': ('django.db.models.fields.TextField', [], {})
},
'layerindex.machine': {
'Meta': {'object_name': 'Machine'},
'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
},
'layerindex.recipe': {
'Meta': {'object_name': 'Recipe'},
'bbclassextend': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'provides': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'pv': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'summary': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'})
},
'layerindex.recipechange': {
'Meta': {'object_name': 'RecipeChange'},
'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
'changeset': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.RecipeChangeset']"}),
'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'recipe': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.Recipe']"}),
'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
'summary': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
},
'layerindex.recipechangeset': {
'Meta': {'object_name': 'RecipeChangeset'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"})
},
'layerindex.recipefiledependency': {
'Meta': {'object_name': 'RecipeFileDependency'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.LayerBranch']"}),
'path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
}
}
complete_apps = ['layerindex']

View File

@ -303,3 +303,43 @@ class BBClass(models.Model):
def __unicode__(self):
return '%s (%s)' % (self.name, self.layerbranch.layer.name)
class RecipeChangeset(models.Model):
user = models.ForeignKey(User)
name = models.CharField(max_length=255)
def __unicode__(self):
return '%s' % (self.name)
class RecipeChange(models.Model):
RECIPE_VARIABLE_MAP = {
'summary': 'SUMMARY',
'description': 'DESCRIPTION',
'section': 'SECTION',
'license': 'LICENSE',
'homepage': 'HOMEPAGE',
'bugtracker': 'BUGTRACKER',
}
changeset = models.ForeignKey(RecipeChangeset)
recipe = models.ForeignKey(Recipe, related_name='+')
summary = models.CharField(max_length=100, blank=True)
description = models.TextField(blank=True)
section = models.CharField(max_length=100, blank=True)
license = models.CharField(max_length=100, blank=True)
homepage = models.URLField("Homepage URL", blank=True)
bugtracker = models.URLField("Bug tracker URL", blank=True)
def changed_fields(self, mapped = False):
res = {}
for field in self._meta.fields:
if not field.name in ['id', 'changeset', 'recipe']:
value = getattr(self, field.name)
if value:
if mapped:
res[self.RECIPE_VARIABLE_MAP[field.name]] = value
else:
res[field.name] = value
return res

View File

@ -90,6 +90,14 @@ def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout
return (tinfoil, tempdir)
def checkout_layer_branch(layerbranch, repodir):
if layerbranch.actual_branch:
branchname = layerbranch.actual_branch
else:
branchname = layerbranch.branch.name
out = utils.runcmd("git checkout origin/%s" % branchname, repodir)
out = utils.runcmd("git clean -f -x", repodir)
def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch):
# Parse layer.conf files for this layer and its dependencies
# This is necessary not just because BBPATH needs to be set in order
@ -109,3 +117,16 @@ def setup_layer(config_data, fetchdir, layerdir, layer, layerbranch):
config_data_copy.delVar('LAYERDIR')
return config_data_copy
def get_var_files(fn, varlist, d):
import bb.cache
varfiles = {}
envdata = bb.cache.Cache.loadDataFull(fn, [], d)
for v in varlist:
history = envdata.varhistory.get_variable_files(v)
if history:
actualfile = history[-1]
else:
actualfile = None
varfiles[v] = actualfile
return varfiles

View File

@ -164,3 +164,9 @@ padding: 8px;
.muted a {
color: #66B8E0;
}
.search-form-table {
border-spacing: 2px;
border-collapse: separate;
padding-bottom: 5px;
}

View File

@ -7,8 +7,8 @@
from django.conf.urls.defaults import *
from django.views.generic import TemplateView, DetailView, ListView
from django.views.defaults import page_not_found
from layerindex.models import LayerItem, Recipe
from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, switch_branch_view, HistoryListView, EditProfileFormView, DuplicatesView
from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, switch_branch_view, HistoryListView, EditProfileFormView, DuplicatesView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView
from layerindex.models import LayerItem, Recipe, RecipeChangeset
urlpatterns = patterns('',
url(r'^$',
@ -59,6 +59,28 @@ urlpatterns = patterns('',
template_name='layerindex/recipedetail.html'),
name='recipe'),
url(r'^layer/(?P<name>[-\w]+)/publish/$', 'layerindex.views.publish', name="publish"),
url(r'^bulkchange/$',
BulkChangeView.as_view(
template_name='layerindex/bulkchange.html'),
name="bulk_change"),
url(r'^bulkchange/(?P<pk>\d+)/search/$',
BulkChangeSearchView.as_view(
template_name='layerindex/bulkchangesearch.html'),
name="bulk_change_search"),
url(r'^bulkchange/(?P<pk>\d+)/edit/$',
bulk_change_edit_view, {'template_name': 'layerindex/bulkchangeedit.html'}, name="bulk_change_edit"),
url(r'^bulkchange/(?P<pk>\d+)/review/$',
DetailView.as_view(
model=RecipeChangeset,
context_object_name='changeset',
template_name='layerindex/bulkchangereview.html'),
name="bulk_change_review"),
url(r'^bulkchange/(?P<pk>\d+)/patches/$',
bulk_change_patch_view, name="bulk_change_patches"),
url(r'^bulkchange/(?P<pk>\d+)/delete/$',
BulkChangeDeleteView.as_view(
template_name='layerindex/deleteconfirm.html'),
name="bulk_change_delete"),
url(r'^branch/(?P<slug>[-\w]+)/$',
switch_branch_view, name="switch_branch"),
url(r'^raw/recipes.txt$',

View File

@ -6,14 +6,14 @@
from django.shortcuts import get_object_or_404, render
from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse, reverse_lazy
from django.core.exceptions import PermissionDenied
from django.template import RequestContext
from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Recipe, Machine, BBClass
from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Recipe, Machine, BBClass, RecipeChange, RecipeChangeset
from datetime import datetime
from django.views.generic import TemplateView, DetailView, ListView
from django.views.generic.edit import UpdateView
from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm, EditProfileForm
from django.views.generic.edit import CreateView, DeleteView, UpdateView
from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm, EditProfileForm, RecipeChangesetForm, AdvancedRecipeSearchForm, BulkChangeEditFormSet
from django.db import transaction
from django.contrib.auth.models import User, Permission
from django.db.models import Q, Count
@ -65,7 +65,7 @@ def delete_layernote_view(request, template_name, slug, pk):
return render(request, template_name, {
'object': layernote,
'object_type': layernote._meta.verbose_name,
'return_url': layeritem.get_absolute_url()
'cancel_url': layeritem.get_absolute_url()
})
def delete_layer_view(request, template_name, slug):
@ -79,7 +79,7 @@ def delete_layer_view(request, template_name, slug):
return render(request, template_name, {
'object': layeritem,
'object_type': layeritem._meta.verbose_name,
'return_url': layeritem.get_absolute_url()
'cancel_url': layeritem.get_absolute_url()
})
def edit_layer_view(request, template_name, slug=None):
@ -174,6 +174,52 @@ def edit_layer_view(request, template_name, slug=None):
'deplistlayers': deplistlayers,
})
def bulk_change_edit_view(request, template_name, pk):
changeset = get_object_or_404(RecipeChangeset, pk=pk)
if request.method == 'POST':
formset = BulkChangeEditFormSet(request.POST, queryset=changeset.recipechange_set.all())
if formset.is_valid():
for form in formset:
form.clear_same_values()
formset.save()
return HttpResponseRedirect(reverse('bulk_change_review', args=(changeset.id,)))
else:
formset = BulkChangeEditFormSet(queryset=changeset.recipechange_set.all())
return render(request, template_name, {
'formset': formset,
})
def bulk_change_patch_view(request, pk):
import os
import os.path
import utils
changeset = get_object_or_404(RecipeChangeset, pk=pk)
# FIXME this couples the web server and machine running the update script together,
# but given that it's a separate script the way is open to decouple them in future
try:
ret = utils.runcmd('python bulkchange.py %d %s' % (int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
if ret:
fn = ret.splitlines()[-1]
if os.path.exists(fn):
if fn.endswith('.tar.gz'):
mimetype = 'application/x-gzip'
else:
mimetype = 'text/x-diff'
response = HttpResponse(mimetype=mimetype)
response['Content-Disposition'] = 'attachment; filename="%s"' % os.path.basename(fn)
with open(fn, "rb") as f:
data = f.read()
response.write(data)
os.remove(fn)
return response
return HttpResponse('No patch data generated', content_type='text/plain')
except Exception as e:
return HttpResponse('Failed to generate patches: %s' % e, content_type='text/plain')
# FIXME better error handling
def _check_branch(request):
branchname = request.GET.get('branch', '')
if branchname:
@ -323,6 +369,137 @@ class DuplicatesView(TemplateView):
context['classes'] = self.get_classes()
return context
class AdvancedRecipeSearchView(ListView):
context_object_name = 'recipe_list'
paginate_by = 50
def get_queryset(self):
field = self.request.GET.get('field', '')
if field:
search_form = AdvancedRecipeSearchForm(self.request.GET)
if not search_form.is_valid():
return Recipe.objects.none()
match_type = self.request.GET.get('match_type', '')
if match_type == 'B':
value = ''
else:
value = self.request.GET.get('value', '')
if value or match_type == 'B':
if match_type == 'C' or match_type == 'N':
query = Q(**{"%s__icontains" % field: value})
else:
query = Q(**{"%s" % field: value})
queryset = Recipe.objects.filter(layerbranch__branch__name=self.request.session.get('branch', 'master'))
layer = self.request.GET.get('layer', '')
if layer:
queryset = queryset.filter(layerbranch__layer=layer)
if match_type == 'N':
# Exclude blank as well
queryset = queryset.exclude(Q(**{"%s" % field: ''})).exclude(query)
else:
queryset = queryset.filter(query)
return queryset.order_by('pn', 'layerbranch__layer')
return Recipe.objects.none()
def get_context_data(self, **kwargs):
context = super(AdvancedRecipeSearchView, self).get_context_data(**kwargs)
if self.request.GET.get('field', ''):
searched = True
search_form = AdvancedRecipeSearchForm(self.request.GET)
else:
searched = False
search_form = AdvancedRecipeSearchForm()
context['search_form'] = search_form
context['searched'] = searched
return context
class BulkChangeView(CreateView):
model = RecipeChangeset
form_class = RecipeChangesetForm
@method_decorator(login_required)
def dispatch(self, request, *args, **kwargs):
return super(BulkChangeView, self).dispatch(request, *args, **kwargs)
def form_valid(self, form):
if not self.request.user.is_authenticated():
raise PermissionDenied
obj = form.save(commit=False)
obj.user = self.request.user
obj.save()
return HttpResponseRedirect(reverse('bulk_change_search', args=(obj.id,)))
def get_context_data(self, **kwargs):
context = super(BulkChangeView, self).get_context_data(**kwargs)
context['changesets'] = RecipeChangeset.objects.filter(user=self.request.user)
return context
class BulkChangeSearchView(AdvancedRecipeSearchView):
def get(self, request, *args, **kwargs):
self.changeset = get_object_or_404(RecipeChangeset, pk=kwargs['pk'])
if self.changeset.user != request.user:
raise PermissionDenied
return super(BulkChangeSearchView, self).get(request, *args, **kwargs)
def post(self, request, *args, **kwargs):
if not request.user.is_authenticated():
raise PermissionDenied
changeset = get_object_or_404(RecipeChangeset, pk=kwargs['pk'])
if changeset.user != request.user:
raise PermissionDenied
def add_recipes(recipes):
for recipe in recipes:
if not changeset.recipechange_set.filter(recipe=recipe):
change = RecipeChange()
change.changeset = changeset
change.recipe = recipe
change.save()
if 'add_selected' in request.POST:
id_list = request.POST.getlist('selecteditems')
id_list = [int(i) for i in id_list if i.isdigit()]
recipes = Recipe.objects.filter(id__in=id_list)
add_recipes(recipes)
elif 'add_all' in request.POST:
add_recipes(self.get_queryset())
elif 'remove_all' in request.POST:
changeset.recipechange_set.all().delete()
return self.get(request, *args, **kwargs)
def get_context_data(self, **kwargs):
context = super(BulkChangeSearchView, self).get_context_data(**kwargs)
context['changeset'] = self.changeset
return context
class BaseDeleteView(DeleteView):
def get_context_data(self, **kwargs):
context = super(BaseDeleteView, self).get_context_data(**kwargs)
obj = context.get('object', None)
if obj:
context['object_type'] = obj._meta.verbose_name
cancel = self.request.GET.get('cancel', '')
if cancel:
context['cancel_url'] = reverse_lazy(cancel, args=(obj.pk,))
return context
class BulkChangeDeleteView(BaseDeleteView):
model = RecipeChangeset
success_url = reverse_lazy('bulk_change')
def get_queryset(self):
qs = super(BulkChangeDeleteView, self).get_queryset()
return qs.filter(user=self.request.user)
class MachineSearchView(ListView):
context_object_name = 'machine_list'
paginate_by = 50

View File

@ -69,6 +69,7 @@
<b class="caret"></b>
</a>
<ul class="dropdown-menu">
<li><a href="{% url bulk_change %}">Bulk Change</a></li>
<li><a href="{% url duplicates %}">Duplicates</a></li>
</ul>
</li>

View File

@ -0,0 +1,69 @@
{% extends "base.html" %}
{% load i18n %}
{% comment %}
layerindex-web - bulk change page template
Copyright (C) 2013 Intel Corporation
Licensed under the MIT license, see COPYING.MIT for details
{% endcomment %}
<!--
{% block title_append %} - bulk change{% endblock %}
-->
{% block content %}
{% autoescape on %}
<h2>Bulk change</h2>
<p>This tool allows you to update the value of "meta" variables (such as
DESCRIPTION and LICENSE) on more than one recipe at once, and then
generate a patch for these changes which can be submitted for merging.</p>
<p>To get started, your changes will need to be associated with a changeset.</p>
{% if changesets %}
<h3>Select an existing changeset</h3>
<ul>
{% for changeset in changesets %}
<li><a href="{% url bulk_change_search changeset.id %}">{{ changeset.name }}</a></li>
{% endfor %}
</ul>
{% endif %}
<h3>Create a new changeset</h3>
<form class="form-inline" method="POST">
{% csrf_token %}
{% for hidden in form.hidden_fields %}
{{ hidden }}
{% endfor %}
{% for field in form.visible_fields %}
{% if field.errors %}
<div class="control-group alert alert-error">
{{ field.errors }}
{% endif %}
<div class="control-group">
{{ field.label_tag }}
{{ field }}
<span class="help-inline custom-help">
{{ field.help_text }}
</span>
{% if field.errors %}
</div>
{% endif %}
</div>
{% endfor %}
<div class="control-group">
<div class="controls">
<button type="submit" class="btn">Create</button>
</div>
</div>
</form>
{% endautoescape %}
{% endblock %}

View File

@ -0,0 +1,54 @@
{% extends "base.html" %}
{% load i18n %}
{% comment %}
layerindex-web - bulk change edit page template
Copyright (C) 2013 Intel Corporation
Licensed under the MIT license, see COPYING.MIT for details
{% endcomment %}
<!--
{% block title_append %} - bulk change{% endblock %}
-->
{% block content %}
{% autoescape on %}
<h2>Edit recipe fields</h2>
<form method="POST">
{{ formset.non_form_errors }}
{{ formset.management_form }}
{% csrf_token %}
{% for form in formset %}
{% for hidden in form.hidden_fields %}
{{ hidden }}
{% endfor %}
<h3>{{ form.instance.recipe.filename }}</h3>
{% for field in form.visible_fields %}
{% if field.errors %}
<div class="control-group alert alert-error">
{{ field.errors }}
{% endif %}
<div class="control-group formfields">
<div class="control-label">
{{ field.label_tag }}
</div>
<div class="controls">
{{ field }}
<span class="help-inline custom-help">
{{ field.help_text }}
</span>
</div>
</div>
{% endfor %}
{% endfor %}
<input type="submit" class="btn" name="save" value="Save"></input>
</form>
{% endautoescape %}
{% endblock %}

View File

@ -0,0 +1,47 @@
{% extends "base.html" %}
{% load i18n %}
{% comment %}
layerindex-web - bulk change result page template
Copyright (C) 2013 Intel Corporation
Licensed under the MIT license, see COPYING.MIT for details
{% endcomment %}
<!--
{% block title_append %} - bulk change{% endblock %}
-->
{% block content %}
{% autoescape on %}
<h2>{{ changeset.name }}</h2>
{% regroup changeset.recipechange_set.all by recipe.layerbranch.layer as changeset_recipes %}
<ul>
{% for layer in changeset_recipes %}
<li>{{ layer.grouper }}
<ul>
{% for change in layer.list %}
<li>
{{ change.recipe.filename }}
<ul>
{% for field in change.changed_fields %}
<li>{{ field }}</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
<a href="{% url bulk_change_search changeset.id %}" class="btn">Add recipes</a>
<a href="{% url bulk_change_edit changeset.id %}" class="btn">Edit</a>
<a href="{% url bulk_change_patches changeset.id %}" class="btn">Get patches</a>
<a href="{% url bulk_change_delete changeset.id %}?cancel=bulk_change_review" class="btn">Delete</a>
{% endautoescape %}
{% endblock %}

View File

@ -0,0 +1,153 @@
{% extends "base.html" %}
{% load i18n %}
{% comment %}
layerindex-web - bulk change search page template
Copyright (C) 2013 Intel Corporation
Licensed under the MIT license, see COPYING.MIT for details
{% endcomment %}
<!--
{% block title_append %} - bulk change{% endblock %}
-->
{% block content %}
{% autoescape on %}
<h2>Add recipes to changeset</h2>
<div class="row-fluid">
<div class="span9">
<div class="row-fluid">
<form id="search-form" class="form-inline" method="GET">
<table class="search-form-table">
<tbody>
<tr>
<td></td>
<td>
{{ search_form.field.errors }}
{{ search_form.match_type.errors }}
{{ search_form.value.errors }}
</td>
</tr>
<tr>
<td>Field:</td>
<td>
{{ search_form.field }}
{{ search_form.match_type }}
{{ search_form.value }}
</td>
</tr>
<tr>
<td></td>
<td>
{{ search_form.layer.errors }}
</td>
</tr>
<tr>
<td>Layer:</td>
<td>{{ search_form.layer }}</td>
</tr>
</tbody>
</table>
<button class="btn" type="submit">Search</button>
</form>
</div>
{% if recipe_list %}
<form id="recipe-select-form" method="POST">
{% csrf_token %}
<table class="table table-striped table-bordered recipestable">
<thead>
<tr>
<th></th>
<th>Recipe name</th>
<th>Version</th>
<th class="span9">Description</th>
<th>Layer</th>
</tr>
</thead>
<tbody>
{% for recipe in recipe_list %}
<tr>
<td><input type="checkbox" name="selecteditems" value="{{ recipe.id }}"></input></td>
<td><a href="{% url recipe recipe.id %}">{{ recipe.name }}</a></td>
<td>{{ recipe.pv }}</td>
<td>{{ recipe.short_desc }}</td>
<td><a href="{% url layer_item recipe.layerbranch.layer.name %}">{{ recipe.layerbranch.layer.name }}</a></td>
</tr>
{% endfor %}
</tbody>
</table>
<input type="submit" class="btn" name="add_selected" value="Add selected"></input>
<input type="submit" class="btn" name="add_all" value="Add all"></input>
{% if is_paginated %}
{% load pagination %}
{% pagination page_obj %}
{% endif %}
{% else %}
{% if searched %}
<p>No matching recipes in database.</p>
{% endif %}
{% endif %}
</div>
<div class="span3">
{% if changeset %}
<div class="well">
<p>{{ changeset.name }}</p>
{% if changeset.recipechange_set.all %}
<small>
{% regroup changeset.recipechange_set.all by recipe.layerbranch.layer as changeset_recipes %}
<ul>
{% for layer in changeset_recipes %}
<li>{{ layer.grouper }}
<ul>
{% for change in layer.list %}
<li>{{ change.recipe.filename }}</li>
{% endfor %}
</ul>
</li>
{% endfor %}
</ul>
</small>
<input type="submit" class="btn" name="remove_all" value="Remove all"></input>
<a href="{% url bulk_change_edit changeset.id %}" class="btn">Edit</a>
{% endif %}
<a href="{% url bulk_change_delete changeset.id %}?cancel=bulk_change_search" class="btn">Delete</a>
</div>
{% endif %}
</form>
</div>
</div>
{% endautoescape %}
{% endblock %}
{% block scripts %}
<script>
enable_value_field = function() {
if($('#id_match_type').val() == 'B')
$('#id_value').prop('disabled', true);
else
$('#id_value').prop('disabled', false);
}
$(document).ready(function() {
$('#id_match_type').change(enable_value_field)
enable_value_field()
});
</script>
{% endblock %}

View File

@ -26,7 +26,7 @@
<form action="" method="post">
{% csrf_token %}
<input type="submit" value="Delete" class='btn btn-warning' />
<a href="{{ return_url }}" class='btn'>Cancel</a>
<a href="{{ cancel_url }}" class='btn'>Cancel</a>
</form>
{% endautoescape %}