Collect bbappends and bbclasses for each layer

Collect bbappend/bbclass info during the update process and display it
on the layer detail page.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

TODO

@@ -27,6 +27,5 @@ Later:
 * Create simple script to check for unlisted layer subdirectories in all repos
 * Move import script to utils/ subdir
 * Are we going to need some way of indicating forks of another layer? (would rather avoid having forks altogether...)
-* Track bbappends in each layer
 * Auto-detect more values from github pages?
 * Ability for submitters to get email notification about publication?

layerindex/admin.py

@@ -62,6 +62,24 @@ class MachineAdmin(admin.ModelAdmin):
     def has_delete_permission(self, request, obj=None):
         return False
 
+class BBAppendAdmin(admin.ModelAdmin):
+    search_fields = ['filename']
+    list_filter = ['layerbranch__layer__name', 'layerbranch__branch__name']
+    readonly_fields = BBAppend._meta.get_all_field_names()
+    def has_add_permission(self, request, obj=None):
+        return False
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+class BBClassAdmin(admin.ModelAdmin):
+    search_fields = ['name']
+    list_filter = ['layerbranch__layer__name', 'layerbranch__branch__name']
+    readonly_fields = BBClass._meta.get_all_field_names()
+    def has_add_permission(self, request, obj=None):
+        return False
+    def has_delete_permission(self, request, obj=None):
+        return False
+
 admin.site.register(Branch, BranchAdmin)
 admin.site.register(LayerItem, LayerItemAdmin)
 admin.site.register(LayerBranch, LayerBranchAdmin)
@@ -71,3 +89,5 @@ admin.site.register(LayerNote, LayerNoteAdmin)
 admin.site.register(Recipe, RecipeAdmin)
 admin.site.register(RecipeFileDependency)
 admin.site.register(Machine, MachineAdmin)
+admin.site.register(BBAppend, BBAppendAdmin)
+admin.site.register(BBClass, BBClassAdmin)

layerindex/migrations/0004_auto__add_bbclass__add_bbappend.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'BBClass'
+        db.create_table('layerindex_bbclass', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('layerbranch', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['layerindex.LayerBranch'])),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=100)),
+        ))
+        db.send_create_signal('layerindex', ['BBClass'])
+
+        # Adding model 'BBAppend'
+        db.create_table('layerindex_bbappend', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('layerbranch', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['layerindex.LayerBranch'])),
+            ('filename', self.gf('django.db.models.fields.CharField')(max_length=255)),
+            ('filepath', self.gf('django.db.models.fields.CharField')(max_length=255, blank=True)),
+        ))
+        db.send_create_signal('layerindex', ['BBAppend'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'BBClass'
+        db.delete_table('layerindex_bbclass')
+
+        # Deleting model 'BBAppend'
+        db.delete_table('layerindex_bbappend')
+
+
+    models = {
+        'layerindex.bbappend': {
+            'Meta': {'object_name': 'BBAppend'},
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"})
+        },
+        'layerindex.bbclass': {
+            'Meta': {'object_name': 'BBClass'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'layerindex.branch': {
+            'Meta': {'object_name': 'Branch'},
+            'bitbake_branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
+            'sort_priority': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'layerindex.layerbranch': {
+            'Meta': {'object_name': 'LayerBranch'},
+            'branch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Branch']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+            'vcs_last_commit': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'vcs_last_fetch': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'vcs_last_rev': ('django.db.models.fields.CharField', [], {'max_length': '80', 'blank': 'True'}),
+            'vcs_subdir': ('django.db.models.fields.CharField', [], {'max_length': '40', 'blank': 'True'})
+        },
+        'layerindex.layerdependency': {
+            'Meta': {'object_name': 'LayerDependency'},
+            'dependency': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependents_set'", 'to': "orm['layerindex.LayerItem']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'dependencies_set'", 'to': "orm['layerindex.LayerBranch']"})
+        },
+        'layerindex.layeritem': {
+            'Meta': {'object_name': 'LayerItem'},
+            'description': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'index_preference': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'layer_type': ('django.db.models.fields.CharField', [], {'max_length': '1'}),
+            'mailing_list_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '40'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'N'", 'max_length': '1'}),
+            'summary': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'usage_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'vcs_url': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'vcs_web_file_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'vcs_web_tree_base_url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'vcs_web_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'layerindex.layermaintainer': {
+            'Meta': {'object_name': 'LayerMaintainer'},
+            'email': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'responsibility': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'A'", 'max_length': '1'})
+        },
+        'layerindex.layernote': {
+            'Meta': {'object_name': 'LayerNote'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layer': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerItem']"}),
+            'text': ('django.db.models.fields.TextField', [], {})
+        },
+        'layerindex.machine': {
+            'Meta': {'object_name': 'Machine'},
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '255'})
+        },
+        'layerindex.recipe': {
+            'Meta': {'object_name': 'Recipe'},
+            'bbclassextend': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'bugtracker': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'filepath': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'homepage': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.LayerBranch']"}),
+            'license': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'pn': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'provides': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
+            'pv': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'section': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'summary': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'layerindex.recipefiledependency': {
+            'Meta': {'object_name': 'RecipeFileDependency'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'layerbranch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['layerindex.LayerBranch']"}),
+            'path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
+            'recipe': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['layerindex.Recipe']"})
+        }
+    }
+
+    complete_apps = ['layerindex']

layerindex/models.py

@@ -270,3 +270,35 @@ class Machine(models.Model):
 
     def __unicode__(self):
         return '%s (%s)' % (self.name, self.layerbranch.layer.name)
+
+
+class BBAppend(models.Model):
+    layerbranch = models.ForeignKey(LayerBranch)
+    filename = models.CharField(max_length=255)
+    filepath = models.CharField(max_length=255, blank=True)
+
+    class Meta:
+        verbose_name = "Append"
+
+    def vcs_web_url(self):
+        url = self.layerbranch.file_url(os.path.join(self.filepath, self.filename))
+        return url or ''
+
+    def __unicode__(self):
+        return os.path.join(self.filepath, self.filename)
+
+
+class BBClass(models.Model):
+    layerbranch = models.ForeignKey(LayerBranch)
+    name = models.CharField(max_length=100)
+
+    class Meta:
+        verbose_name = "Class"
+        verbose_name_plural = "Classes"
+
+    def vcs_web_url(self):
+        url = self.layerbranch.file_url(os.path.join('classes', "%s.bbclass" % self.name))
+        return url or ''
+
+    def __unicode__(self):
+        return '%s (%s)' % (self.name, self.layerbranch.layer.name)

layerindex/update.py

@@ -59,16 +59,37 @@ def runcmd(cmd,destdir=None,printerr=True):
     return output
 
 
-def split_bb_file_path(recipe_path, subdir_start):
+machine_conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
-    if fnmatch.fnmatch(recipe_path, "*.bb"):
+bbclass_re = re.compile(r'classes/([^/.]*).bbclass$')
-        if subdir_start:
+def detect_file_type(path, subdir_start):
-            filepath = os.path.relpath(os.path.dirname(recipe_path), subdir_start)
+    typename = None
+    if fnmatch.fnmatch(path, "*.bb"):
+        typename = 'recipe'
+    elif fnmatch.fnmatch(path, "*.bbappend"):
+        typename = 'bbappend'
     else:
-            filepath = os.path.dirname(recipe_path)
+        # Check if it's a machine conf file
-        return (filepath, os.path.basename(recipe_path))
+        subpath = path[len(subdir_start):]
-    return (None, None)
+        res = machine_conf_re.match(subpath)
+        if res:
+            typename = 'machine'
+            return (typename, None, res.group(1))
+        else:
+            res = bbclass_re.match(subpath)
+            if res:
+                typename = 'bbclass'
+                return (typename, None, res.group(1))
+
+    if typename == 'recipe' or typename == 'bbappend':
+        if subdir_start:
+            filepath = os.path.relpath(os.path.dirname(path), subdir_start)
+        else:
+            filepath = os.path.dirname(path)
+        return (typename, filepath, os.path.basename(path))
+
+    return (None, None, None)
+
 
-conf_re = re.compile(r'conf/machine/([^/.]*).conf$')
 def check_machine_conf(path, subdir_start):
     subpath = path[len(subdir_start):]
     res = conf_re.match(subpath)
@@ -202,7 +223,7 @@ def main():
 
     from django.core.management import setup_environ
     from django.conf import settings
-    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine
+    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
     from django.db import transaction
     import settings
 
@@ -379,6 +400,8 @@ def main():
             layerdir_start = os.path.normpath(layerdir) + os.sep
             layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
             layermachines = Machine.objects.filter(layerbranch=layerbranch)
+            layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
+            layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
             if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
                 # Check out appropriate branch
                 if not options.nocheckout:
@@ -443,19 +466,21 @@ def main():
                     for d in diff.iter_change_type('D'):
                         path = d.a_blob.path
                         if path.startswith(subdir_start):
-                            (filepath, filename) = split_bb_file_path(path, subdir_start)
+                            (typename, filepath, filename) = detect_file_type(path, subdir_start)
-                            if filename:
+                            if typename == 'recipe':
                                 layerrecipes.filter(filepath=filepath).filter(filename=filename).delete()
-                            else:
+                            elif typename == 'bbappend':
-                                machinename = check_machine_conf(path, subdir_start)
+                                layerappends.filter(filepath=filepath).filter(filename=filename).delete()
-                                if machinename:
+                            elif typename == 'machine':
-                                    layermachines.filter(name=machinename).delete()
+                                layermachines.filter(name=filename).delete()
+                            elif typename == 'bbclass':
+                                layerclasses.filter(name=filename).delete()
 
                     for d in diff.iter_change_type('A'):
                         path = d.b_blob.path
                         if path.startswith(subdir_start):
-                            (filepath, filename) = split_bb_file_path(path, subdir_start)
+                            (typename, filepath, filename) = detect_file_type(path, subdir_start)
-                            if filename:
+                            if typename == 'recipe':
                                 recipe = Recipe()
                                 recipe.layerbranch = layerbranch
                                 recipe.filename = filename
@@ -463,35 +488,43 @@ def main():
                                 update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
                                 recipe.save()
                                 updatedrecipes.add(recipe)
-                            else:
+                            elif typename == 'bbappend':
-                                machinename = check_machine_conf(path, subdir_start)
+                                append = BBAppend()
-                                if machinename:
+                                append.layerbranch = layerbranch
+                                append.filename = filename
+                                append.filepath = filepath
+                                append.save()
+                            elif typename == 'machine':
                                 machine = Machine()
                                 machine.layerbranch = layerbranch
-                                    machine.name = machinename
+                                machine.name = filename
                                 update_machine_conf_file(os.path.join(repodir, path), machine)
                                 machine.save()
+                            elif typename == 'bbclass':
+                                bbclass = BBClass()
+                                bbclass.layerbranch = layerbranch
+                                bbclass.name = filename
+                                bbclass.save()
 
                     dirtyrecipes = set()
                     for d in diff.iter_change_type('M'):
                         path = d.a_blob.path
                         if path.startswith(subdir_start):
-                            (filepath, filename) = split_bb_file_path(path, subdir_start)
+                            (typename, filepath, filename) = detect_file_type(path, subdir_start)
-                            if filename:
+                            if typename == 'recipe':
                                 results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
                                 if results:
                                     recipe = results[0]
                                     update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
                                     recipe.save()
                                     updatedrecipes.add(recipe)
-                            else:
+                            elif typename == 'machine':
-                                machinename = check_machine_conf(path, subdir_start)
+                                results = layermachines.filter(name=filename)
-                                if machinename:
-                                    results = layermachines.filter(name=machinename)
                                 if results:
                                     machine = results[0]
                                     update_machine_conf_file(os.path.join(repodir, path), machine)
                                     machine.save()
+
                             deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
                             for dep in deps:
                                 dirtyrecipes.add(dep.recipe)
@@ -503,26 +536,38 @@ def main():
                     # Collect recipe data from scratch
                     layerrecipes.delete()
                     layermachines.delete()
+                    layerappends.delete()
+                    layerclasses.delete()
                     for root, dirs, files in os.walk(layerdir):
                         if '.git' in dirs:
                             dirs.remove('.git')
                         for f in files:
-                            if fnmatch.fnmatch(f, "*.bb"):
+                            fullpath = os.path.join(root, f)
+                            (typename, _, filename) = detect_file_type(fullpath, layerdir_start)
+                            if typename == 'recipe':
                                 recipe = Recipe()
                                 recipe.layerbranch = layerbranch
                                 recipe.filename = f
                                 recipe.filepath = os.path.relpath(root, layerdir)
                                 update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
                                 recipe.save()
-                            else:
+                            elif typename == 'bbappend':
-                                fullpath = os.path.join(root, f)
+                                append = BBAppend()
-                                machinename = check_machine_conf(fullpath, layerdir_start)
+                                append.layerbranch = layerbranch
-                                if machinename:
+                                append.filename = f
+                                append.filepath = os.path.relpath(root, layerdir)
+                                append.save()
+                            elif typename == 'machine':
                                 machine = Machine()
                                 machine.layerbranch = layerbranch
-                                    machine.name = machinename
+                                machine.name = filename
                                 update_machine_conf_file(fullpath, machine)
                                 machine.save()
+                            elif typename == 'bbclass':
+                                bbclass = BBClass()
+                                bbclass.layerbranch = layerbranch
+                                bbclass.name = filename
+                                bbclass.save()
 
                 # Save repo info
                 layerbranch.vcs_last_rev = topcommit.hexsha

layerindex/views.py

@@ -245,6 +245,8 @@ class LayerDetailView(DetailView):
         layerbranch = layer.get_layerbranch(self.request.session.get('branch', 'master'))
         context['layerbranch'] = layerbranch
         context['machines'] = layerbranch.machine_set.order_by('name')
+        context['appends'] = layerbranch.bbappend_set.order_by('filename')
+        context['classes'] = layerbranch.bbclass_set.order_by('name')
         return context
 
 class LayerReviewDetailView(LayerDetailView):

templates/layerindex/detail.html

@@ -214,6 +214,51 @@
         </div>
         {% endif %}
 
+        {% if appends.count > 0 %}
+        <div class="container-fluid" style="margin-bottom:1em;">
+            <div class="row-fluid">
+                <div class="navbar">
+                    <div class="navbar-inner">
+                            <a class="brand pull-left">{{ layeritem.name }} bbappends</a>
+                    </div>
+                </div>
+
+                <table class="table table-bordered">
+                    <tbody>
+                        {% for append in appends %}
+                            <tr>
+                                <td><a href="{{ append.vcs_web_url }}">{{ append.filename }}</a></td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+             </div>
+         </div>
+        {% endif %}
+
+        {% if classes.count > 0 %}
+        <div class="container-fluid" style="margin-bottom:1em;">
+            <div class="row-fluid">
+                <div class="navbar">
+                    <div class="navbar-inner">
+                            <a class="brand pull-left">{{ layeritem.name }} classes</a>
+                    </div>
+                </div>
+
+                <table class="table table-bordered">
+                    <tbody>
+                        {% for class in classes %}
+                            <tr>
+                                <td><a href="{{ class.vcs_web_url }}">{{ class.name }}</a></td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+             </div>
+         </div>
+        {% endif %}
+
+
 {% endautoescape %}
 
 {% endblock %}