From e591d1820ab35a9df38a92d76b26ea5733169919 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Tue, 6 Nov 2018 09:40:33 +1300 Subject: [PATCH] Track and enable reporting on duplicate inc files It's not too common but there are instances where people have copied .inc files into their own layer and modified them, and if you are using such a layer that could result in unexpected behaviour. In order to get a handle on when this is being done, collect data about all .inc files and show duplicates in the Duplicates screen. Signed-off-by: Paul Eggleton --- layerindex/admin.py | 10 +++++++++ layerindex/migrations/0026_incfile.py | 24 ++++++++++++++++++++ layerindex/models.py | 12 ++++++++++ layerindex/recipeparse.py | 4 +++- layerindex/update_layer.py | 24 +++++++++++++++++++- layerindex/views.py | 11 ++++++++- templates/layerindex/duplicates.html | 32 +++++++++++++++++++++++++-- 7 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 layerindex/migrations/0026_incfile.py diff --git a/layerindex/admin.py b/layerindex/admin.py index fa36663..6db42e7 100644 --- a/layerindex/admin.py +++ b/layerindex/admin.py @@ -168,6 +168,15 @@ class BBClassAdmin(admin.ModelAdmin): def has_delete_permission(self, request, obj=None): return False +class IncFileAdmin(admin.ModelAdmin): + search_fields = ['path'] + list_filter = ['layerbranch__layer__name', 'layerbranch__branch__name'] + readonly_fields = [f.name for f in IncFile._meta.get_fields()] + def has_add_permission(self, request, obj=None): + return False + def has_delete_permission(self, request, obj=None): + return False + class RecipeChangeInline(admin.StackedInline): model = RecipeChange @@ -200,6 +209,7 @@ admin.site.register(Machine, MachineAdmin) admin.site.register(Distro, DistroAdmin) admin.site.register(BBAppend, BBAppendAdmin) admin.site.register(BBClass, BBClassAdmin) +admin.site.register(IncFile, IncFileAdmin) admin.site.register(Patch) admin.site.register(LayerRecipeExtraURL) admin.site.register(RecipeChangeset, RecipeChangesetAdmin) diff --git a/layerindex/migrations/0026_incfile.py b/layerindex/migrations/0026_incfile.py new file mode 100644 index 0000000..27168dd --- /dev/null +++ b/layerindex/migrations/0026_incfile.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.12 on 2018-11-05 19:50 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('layerindex', '0025_update_retcode'), + ] + + operations = [ + migrations.CreateModel( + name='IncFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('path', models.CharField(max_length=255)), + ('layerbranch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='layerindex.LayerBranch')), + ], + ), + ] diff --git a/layerindex/models.py b/layerindex/models.py index 6633150..5292bff 100644 --- a/layerindex/models.py +++ b/layerindex/models.py @@ -755,6 +755,18 @@ class BBClass(models.Model): return '%s (%s)' % (self.name, self.layerbranch.layer.name) +class IncFile(models.Model): + layerbranch = models.ForeignKey(LayerBranch) + path = models.CharField(max_length=255) + + def vcs_web_url(self): + url = self.layerbranch.file_url(self.path) + return url or '' + + def __str__(self): + return '%s (%s)' % (self.path, self.layerbranch.layer.name) + + class RecipeChangeset(models.Model): user = models.ForeignKey(User) name = models.CharField(max_length=255) diff --git a/layerindex/recipeparse.py b/layerindex/recipeparse.py index bad78f4..2e1a251 100644 --- a/layerindex/recipeparse.py +++ b/layerindex/recipeparse.py @@ -126,6 +126,8 @@ def detect_file_type(path, subdir_start): typename = 'recipe' elif fnmatch.fnmatch(path, "*.bbappend"): typename = 'bbappend' + elif fnmatch.fnmatch(path, "*.inc"): + typename = 'incfile' else: # Check if it's a machine conf file subpath = path[len(subdir_start):] @@ -142,7 +144,7 @@ def detect_file_type(path, subdir_start): typename = 'distro' return (typename, None, res.group(1)) - if typename == 'recipe' or typename == 'bbappend': + if typename in ['recipe', 'bbappend', 'incfile']: if subdir_start: filepath = os.path.relpath(os.path.dirname(path), subdir_start) else: diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py index 42b618f..3ae4f65 100644 --- a/layerindex/update_layer.py +++ b/layerindex/update_layer.py @@ -347,7 +347,7 @@ def main(): utils.setup_django() import settings - from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass + from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile from django.db import transaction logger.setLevel(options.loglevel) @@ -421,6 +421,7 @@ def main(): layerdistros = Distro.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerclasses = BBClass.objects.filter(layerbranch=layerbranch) + layerincfiles = IncFile.objects.filter(layerbranch=layerbranch) if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial: # Check out appropriate branch if not options.nocheckout: @@ -585,6 +586,15 @@ def main(): else: logger.warn("Renamed class %s could not be found" % oldpath) other_adds.append(diffitem) + elif oldtypename == 'incfile': + results = layerincfiles.filter(path=os.path.join(oldfilepath, oldfilename)) + if len(results): + logger.debug("Rename inc file %s to %s" % (results[0], newfilename)) + results[0].name = newfilename + results[0].save() + else: + logger.warn("Renamed inc file %s could not be found" % oldpath) + other_adds.append(diffitem) deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=oldpath) for dep in deps: @@ -618,6 +628,8 @@ def main(): layerdistros.filter(name=filename).delete() elif typename == 'bbclass': layerclasses.filter(name=filename).delete() + elif typename == 'incfile': + layerincfiles.filter(path=os.path.join(filepath, filename)).delete() for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds): path = diffitem.b_blob.path @@ -657,6 +669,11 @@ def main(): bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() + elif typename == 'incfile': + incfile = IncFile() + incfile.layerbranch = layerbranch + incfile.path = os.path.join(filepath, filename) + incfile.save() for diffitem in diff.iter_change_type('M'): path = diffitem.b_blob.path @@ -773,6 +790,11 @@ def main(): bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() + elif typename == 'incfile': + incfile = IncFile() + incfile.layerbranch = layerbranch + incfile.path = os.path.relpath(fullpath, layerdir) + incfile.save() for added in layerrecipes_add: # This is good enough without actually parsing the file diff --git a/layerindex/views.py b/layerindex/views.py index 1ce1a22..b9a6afb 100644 --- a/layerindex/views.py +++ b/layerindex/views.py @@ -13,7 +13,7 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidde from django.core.urlresolvers import reverse, reverse_lazy, resolve from django.core.exceptions import PermissionDenied from django.template import RequestContext -from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Update, LayerUpdate, Recipe, Machine, Distro, BBClass, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe, StaticBuildDep, DynamicBuildDep +from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Update, LayerUpdate, Recipe, Machine, Distro, BBClass, IncFile, BBAppend, RecipeChange, RecipeChangeset, ClassicRecipe, StaticBuildDep, DynamicBuildDep from datetime import datetime from django.views.generic import TemplateView, DetailView, ListView from django.views.generic.edit import CreateView, DeleteView, UpdateView @@ -555,11 +555,20 @@ class DuplicatesView(TemplateView): qs = init_qs.all().filter(name__in=[item['name'] for item in dupes]).order_by('name', 'layerbranch__layer') return qs + def get_incfiles(self, layer_ids): + init_qs = IncFile.objects.filter(layerbranch__branch__name=self.kwargs['branch']) + if layer_ids: + init_qs = init_qs.filter(layerbranch__layer__in=layer_ids) + dupes = init_qs.values('path').annotate(Count('layerbranch', distinct=True)).filter(layerbranch__count__gt=1) + qs = init_qs.all().filter(path__in=[item['path'] for item in dupes]).order_by('path', 'layerbranch__layer') + return qs + def get_context_data(self, **kwargs): layer_ids = [int(i) for i in self.request.GET.getlist('l')] context = super(DuplicatesView, self).get_context_data(**kwargs) context['recipes'] = self.get_recipes(layer_ids) context['classes'] = self.get_classes(layer_ids) + context['incfiles'] = self.get_incfiles(layer_ids) context['url_branch'] = self.kwargs['branch'] context['this_url_name'] = resolve(self.request.path_info).url_name context['layers'] = LayerBranch.objects.filter(branch__name=self.kwargs['branch']).filter(layer__status__in=['P', 'X']).order_by( 'layer__name') diff --git a/templates/layerindex/duplicates.html b/templates/layerindex/duplicates.html index bf9074a..71c3922 100644 --- a/templates/layerindex/duplicates.html +++ b/templates/layerindex/duplicates.html @@ -110,7 +110,7 @@ {% else %} -

No duplicate recipes in database.

+

No matching duplicate recipes in database.

{% endif %} @@ -139,7 +139,35 @@ {% else %} -

No duplicate classes in database.

+

No matching duplicate classes in database.

+{% endif %} + + + +
+
+

Duplicate include files

+{% if incfiles %} +

Include files with the same name in different layers:

+ + + + + + + + + + {% for incfile in incfiles %} + + + + + {% endfor %} + +
Include fileLayer
{{ incfile.path }}{{ incfile.layerbranch.layer.name }}
+{% else %} +

No matching duplicate include files in database.

{% endif %}