Track and enable reporting on duplicate inc files

It's not too common but there are instances where people have copied
.inc files into their own layer and modified them, and if you are using
such a layer that could result in unexpected behaviour. In order to get
a handle on when this is being done, collect data about all .inc files
and show duplicates in the Duplicates screen.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

layerindex/admin.py

@@ -168,6 +168,15 @@ class BBClassAdmin(admin.ModelAdmin):
     def has_delete_permission(self, request, obj=None):
         return False
 
+class IncFileAdmin(admin.ModelAdmin):
+    search_fields = ['path']
+    list_filter = ['layerbranch__layer__name', 'layerbranch__branch__name']
+    readonly_fields = [f.name for f in IncFile._meta.get_fields()]
+    def has_add_permission(self, request, obj=None):
+        return False
+    def has_delete_permission(self, request, obj=None):
+        return False
+
 class RecipeChangeInline(admin.StackedInline):
     model = RecipeChange
 
@@ -200,6 +209,7 @@ admin.site.register(Machine, MachineAdmin)
 admin.site.register(Distro, DistroAdmin)
 admin.site.register(BBAppend, BBAppendAdmin)
 admin.site.register(BBClass, BBClassAdmin)
+admin.site.register(IncFile, IncFileAdmin)
 admin.site.register(Patch)
 admin.site.register(LayerRecipeExtraURL)
 admin.site.register(RecipeChangeset, RecipeChangesetAdmin)

layerindex/migrations/0026_incfile.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.12 on 2018-11-05 19:50
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layerindex', '0025_update_retcode'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='IncFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.CharField(max_length=255)),
+                ('layerbranch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.LayerBranch')),
+            ],
+        ),
+    ]

layerindex/models.py

@@ -755,6 +755,18 @@ class BBClass(models.Model):
         return '%s (%s)' % (self.name, self.layerbranch.layer.name)
 
 
+class IncFile(models.Model):
+    layerbranch = models.ForeignKey(LayerBranch)
+    path = models.CharField(max_length=255)
+
+    def vcs_web_url(self):
+        url = self.layerbranch.file_url(self.path)
+        return url or ''
+
+    def __str__(self):
+        return '%s (%s)' % (self.path, self.layerbranch.layer.name)
+
+
 class RecipeChangeset(models.Model):
     user = models.ForeignKey(User)
     name = models.CharField(max_length=255)

layerindex/recipeparse.py

@@ -126,6 +126,8 @@ def detect_file_type(path, subdir_start):
         typename = 'recipe'
     elif fnmatch.fnmatch(path, "*.bbappend"):
         typename = 'bbappend'
+    elif fnmatch.fnmatch(path, "*.inc"):
+        typename = 'incfile'
     else:
         # Check if it's a machine conf file
         subpath = path[len(subdir_start):]
@@ -142,7 +144,7 @@ def detect_file_type(path, subdir_start):
             typename = 'distro'
             return (typename, None, res.group(1))
 
-    if typename == 'recipe' or typename == 'bbappend':
+    if typename in ['recipe', 'bbappend', 'incfile']:
         if subdir_start:
             filepath = os.path.relpath(os.path.dirname(path), subdir_start)
         else:

layerindex/update_layer.py

@@ -347,7 +347,7 @@ def main():
 
     utils.setup_django()
     import settings
-    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass
+    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile
     from django.db import transaction
 
     logger.setLevel(options.loglevel)
@@ -421,6 +421,7 @@ def main():
             layerdistros = Distro.objects.filter(layerbranch=layerbranch)
             layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
             layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
+            layerincfiles = IncFile.objects.filter(layerbranch=layerbranch)
             if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial:
                 # Check out appropriate branch
                 if not options.nocheckout:
@@ -585,6 +586,15 @@ def main():
                                 else:
                                     logger.warn("Renamed class %s could not be found" % oldpath)
                                     other_adds.append(diffitem)
+                            elif oldtypename == 'incfile':
+                                results = layerincfiles.filter(path=os.path.join(oldfilepath, oldfilename))
+                                if len(results):
+                                    logger.debug("Rename inc file %s to %s" % (results[0], newfilename))
+                                    results[0].name = newfilename
+                                    results[0].save()
+                                else:
+                                    logger.warn("Renamed inc file %s could not be found" % oldpath)
+                                    other_adds.append(diffitem)
 
                             deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath)
                             for dep in deps:
@@ -618,6 +628,8 @@ def main():
                                 layerdistros.filter(name=filename).delete()
                             elif typename == 'bbclass':
                                 layerclasses.filter(name=filename).delete()
+                            elif typename == 'incfile':
+                                layerincfiles.filter(path=os.path.join(filepath, filename)).delete()
 
                     for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds):
                         path = diffitem.b_blob.path
@@ -657,6 +669,11 @@ def main():
                                 bbclass.layerbranch = layerbranch
                                 bbclass.name = filename
                                 bbclass.save()
+                            elif typename == 'incfile':
+                                incfile = IncFile()
+                                incfile.layerbranch = layerbranch
+                                incfile.path = os.path.join(filepath, filename)
+                                incfile.save()
 
                     for diffitem in diff.iter_change_type('M'):
                         path = diffitem.b_blob.path
@@ -773,6 +790,11 @@ def main():
                                 bbclass.layerbranch = layerbranch
                                 bbclass.name = filename
                                 bbclass.save()
+                            elif typename == 'incfile':
+                                incfile = IncFile()
+                                incfile.layerbranch = layerbranch
+                                incfile.path = os.path.relpath(fullpath, layerdir)
+                                incfile.save()
 
                 for added in layerrecipes_add:
                     # This is good enough without actually parsing the file

layerindex/views.py

@@ -13,7 +13,7 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidde
 from django.core.urlresolvers import reverse, reverse_lazy, resolve
 from django.core.exceptions import PermissionDenied
 from django.template import RequestContext
-from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Update, LayerUpdate, Recipe, Machine, Distro, BBClass, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe, StaticBuildDep, DynamicBuildDep
+from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Update, LayerUpdate, Recipe, Machine, Distro, BBClass, IncFile, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe, StaticBuildDep, DynamicBuildDep
 from datetime import datetime
 from django.views.generic import TemplateView, DetailView, ListView
 from django.views.generic.edit import CreateView, DeleteView, UpdateView
@@ -555,11 +555,20 @@ class DuplicatesView(TemplateView):
         qs = init_qs.all().filter(name__in=[item['name'] for item in dupes]).order_by('name', 'layerbranch__layer')
         return qs
 
+    def get_incfiles(self, layer_ids):
+        init_qs = IncFile.objects.filter(layerbranch__branch__name=self.kwargs['branch'])
+        if layer_ids:
+            init_qs = init_qs.filter(layerbranch__layer__in=layer_ids)
+        dupes = init_qs.values('path').annotate(Count('layerbranch', distinct=True)).filter(layerbranch__count__gt=1)
+        qs = init_qs.all().filter(path__in=[item['path'] for item in dupes]).order_by('path', 'layerbranch__layer')
+        return qs
+
     def get_context_data(self, **kwargs):
         layer_ids = [int(i) for i in self.request.GET.getlist('l')]
         context = super(DuplicatesView, self).get_context_data(**kwargs)
         context['recipes'] = self.get_recipes(layer_ids)
         context['classes'] = self.get_classes(layer_ids)
+        context['incfiles'] = self.get_incfiles(layer_ids)
         context['url_branch'] = self.kwargs['branch']
         context['this_url_name'] = resolve(self.request.path_info).url_name
         context['layers'] = LayerBranch.objects.filter(branch__name=self.kwargs['branch']).filter(layer__status__in=['P', 'X']).order_by( 'layer__name')

templates/layerindex/duplicates.html

@@ -110,7 +110,7 @@
                     </tbody>
                 </table>
 {% else %}
-    <p>No duplicate recipes in database.</p>
+    <p>No matching duplicate recipes in database.</p>
 {% endif %}
             </div>
         </div>
@@ -139,7 +139,35 @@
                     </tbody>
                 </table>
 {% else %}
-    <p>No duplicate classes in database.</p>
+    <p>No matching duplicate classes in database.</p>
+{% endif %}
+            </div>
+        </div>
+
+        <div class="row">
+            <div class="col-md-12">
+                <h2>Duplicate include files</h2>
+{% if incfiles %}
+                <p>Include files with the same name in different layers:</p>
+                <table class="table table-striped table-bordered recipestable">
+                    <thead>
+                        <tr>
+                            <th>Include file</th>
+                            <th>Layer</th>
+                        </tr>
+                    </thead>
+
+                    <tbody>
+                        {% for incfile in incfiles %}
+                            <tr>
+                                <td><a href="{{ incfile.vcs_web_url }}">{{ incfile.path }}</a></td>
+                                <td><a href="{% url 'layer_item' 'master' incfile.layerbranch.layer.name %}">{{ incfile.layerbranch.layer.name }}</a></td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+{% else %}
+    <p>No matching duplicate include files in database.</p>
 {% endif %}
             </div>
         </div>