From 8dbe8d09b93127b81d3ae962e383bc62ed3387b9 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Tue, 12 Nov 2019 14:09:00 +1300 Subject: [PATCH] Add recipe dependencies tool Add an extra tool that lets you view all of the recipe dependencies in a layer. There is also a mode that shows only cross-layer dependencies, which can be useful to find dependencies on recipes in other layers that aren't declared in the layer's dependencies (or conversely where a layer dependency is no longer necessary). Signed-off-by: Paul Eggleton --- layerindex/admin.py | 1 + layerindex/forms.py | 13 ++ .../migrations/0044_extendedprovides.py | 46 ++++ layerindex/models.py | 8 + layerindex/recipeparse.py | 17 +- layerindex/update_layer.py | 2 + layerindex/urls.py | 6 +- layerindex/views.py | 116 +++++++++- templates/base.html | 1 + templates/layerindex/recipedeps.html | 209 ++++++++++++++++++ 10 files changed, 415 insertions(+), 4 deletions(-) create mode 100644 layerindex/migrations/0044_extendedprovides.py create mode 100644 templates/layerindex/recipedeps.html diff --git a/layerindex/admin.py b/layerindex/admin.py index b89ef56..9ef781d 100644 --- a/layerindex/admin.py +++ b/layerindex/admin.py @@ -234,6 +234,7 @@ admin.site.register(LayerUpdate, LayerUpdateAdmin) admin.site.register(PackageConfig, PackageConfigAdmin) admin.site.register(StaticBuildDep, StaticBuildDepAdmin) admin.site.register(DynamicBuildDep, DynamicBuildDepAdmin) +admin.site.register(ExtendedProvide) admin.site.register(Source, SourceAdmin) admin.site.register(Recipe, RecipeAdmin) admin.site.register(RecipeFileDependency) diff --git a/layerindex/forms.py b/layerindex/forms.py index 51583c6..4d8eb44 100644 --- a/layerindex/forms.py +++ b/layerindex/forms.py @@ -373,3 +373,16 @@ class BranchComparisonForm(StyledForm): if cleaned_data['from_branch'] == cleaned_data['to_branch']: raise forms.ValidationError({'to_branch': 'From and to branches cannot be the same'}) return cleaned_data + + +class RecipeDependenciesForm(StyledForm): + branch = forms.ModelChoiceField(label='Branch', queryset=Branch.objects.none()) + layer = forms.ModelChoiceField(queryset=LayerItem.objects.filter(comparison=False).filter(status__in=['P', 'X']).order_by('name'), required=True) + crosslayer = forms.BooleanField(required=False) + excludelayers = forms.CharField(widget=forms.HiddenInput()) + + def __init__(self, *args, request=None, **kwargs): + super(RecipeDependenciesForm, self).__init__(*args, **kwargs) + qs = Branch.objects.filter(comparison=False, hidden=False).order_by('sort_priority', 'name') + self.fields['branch'].queryset = qs + self.request = request diff --git a/layerindex/migrations/0044_extendedprovides.py b/layerindex/migrations/0044_extendedprovides.py new file mode 100644 index 0000000..5e149ac --- /dev/null +++ b/layerindex/migrations/0044_extendedprovides.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.22 on 2019-11-05 22:51 +from __future__ import unicode_literals + +from django.db import migrations, models + + +def populate_extended_provides(apps, schema_editor): + Branch = apps.get_model('layerindex', 'Branch') + LayerBranch = apps.get_model('layerindex', 'LayerBranch') + Recipe = apps.get_model('layerindex', 'Recipe') + ExtendedProvide = apps.get_model('layerindex', 'ExtendedProvide') + + for branch in Branch.objects.filter(comparison=False): + for layerbranch in LayerBranch.objects.filter(branch=branch): + for recipe in Recipe.objects.filter(layerbranch=layerbranch): + provides = recipe.provides.split() + for extend in recipe.bbclassextend.split(): + if extend == 'native': + provides.append('%s-native' % recipe.pn) + elif extend == 'nativesdk': + provides.append('nativesdk-%s' % recipe.pn) + for provide in provides: + provides, created = ExtendedProvide.objects.get_or_create(name=provide) + if created: + provides.save() + provides.recipes.add(recipe) + + +class Migration(migrations.Migration): + + dependencies = [ + ('layerindex', '0043_recipe_srcrev'), + ] + + operations = [ + migrations.CreateModel( + name='ExtendedProvide', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ('recipes', models.ManyToManyField(to='layerindex.Recipe')), + ], + ), + migrations.RunPython(populate_extended_provides, reverse_code=migrations.RunPython.noop), + ] diff --git a/layerindex/models.py b/layerindex/models.py index 253c725..04888be 100644 --- a/layerindex/models.py +++ b/layerindex/models.py @@ -919,3 +919,11 @@ class PatchDisposition(models.Model): def __str__(self): return '%s - %s' % (self.patch, self.get_disposition_display()) + + +class ExtendedProvide(models.Model): + recipes = models.ManyToManyField(Recipe) + name = models.CharField(max_length=255, unique=True) + + def __str__(self): + return self.name diff --git a/layerindex/recipeparse.py b/layerindex/recipeparse.py index 6748116..72e17e5 100644 --- a/layerindex/recipeparse.py +++ b/layerindex/recipeparse.py @@ -194,4 +194,19 @@ def handle_recipe_depends(recipe, depends, packageconfig_opts): dynamic_build_dependency.save() dynamic_build_dependency.package_configs.add(package_config) dynamic_build_dependency.recipes.add(recipe) - + +def handle_recipe_provides(recipe): + from layerindex.models import ExtendedProvide + + recipe.extendedprovide_set.clear() + provides = recipe.provides.split() + for extend in recipe.bbclassextend.split(): + if extend == 'native': + provides.append('%s-native' % recipe.pn) + elif extend == 'nativesdk': + provides.append('nativesdk-%s' % recipe.pn) + for provide in provides: + provides, created = ExtendedProvide.objects.get_or_create(name=provide) + if created: + provides.save() + provides.recipes.add(recipe) diff --git a/layerindex/update_layer.py b/layerindex/update_layer.py index 1709837..aee5a97 100644 --- a/layerindex/update_layer.py +++ b/layerindex/update_layer.py @@ -150,6 +150,8 @@ def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir, sto recipeparse.handle_recipe_depends(recipe, envdata.getVar('DEPENDS', True) or '', envdata.getVarFlags('PACKAGECONFIG')) + recipeparse.handle_recipe_provides(recipe) + if not skip_patches: # Handle patches collect_patches(recipe, envdata, layerdir_start, stop_on_error) diff --git a/layerindex/urls.py b/layerindex/urls.py index abeb092..6208753 100644 --- a/layerindex/urls.py +++ b/layerindex/urls.py @@ -15,7 +15,7 @@ from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDeta ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, \ UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView, TaskStatusView, \ ComparisonRecipeSelectView, ComparisonRecipeSelectDetailView, task_log_view, task_stop_view, email_test_view, \ - BranchCompareView + BranchCompareView, RecipeDependenciesView from layerindex.models import LayerItem, Recipe, RecipeChangeset from rest_framework import routers from . import restviews @@ -195,6 +195,10 @@ urlpatterns = [ content_type='text/plain', template_name='layerindex/branchcompare_plain.txt'), name='branch_comparison_plain'), + url(r'^recipe_deps/$', + RecipeDependenciesView.as_view( + template_name='layerindex/recipedeps.html'), + name='recipe_deps'), url(r'^ajax/layerchecklist/(?P[-.\w]+)/$', LayerCheckListView.as_view( template_name='layerindex/layerchecklist.html'), diff --git a/layerindex/views.py b/layerindex/views.py index 02c9d07..e9e7193 100644 --- a/layerindex/views.py +++ b/layerindex/views.py @@ -48,14 +48,14 @@ from layerindex.forms import (AdvancedRecipeSearchForm, BulkChangeEditFormSet, EditNoteForm, EditProfileForm, LayerMaintainerFormSet, RecipeChangesetForm, PatchDispositionForm, PatchDispositionFormSet, - BranchComparisonForm) + BranchComparisonForm, RecipeDependenciesForm) from layerindex.models import (BBAppend, BBClass, Branch, ClassicRecipe, Distro, DynamicBuildDep, IncFile, LayerBranch, LayerDependency, LayerItem, LayerMaintainer, LayerNote, LayerUpdate, Machine, Patch, Recipe, RecipeChange, RecipeChangeset, Source, StaticBuildDep, Update, SecurityQuestion, SecurityQuestionAnswer, - UserProfile, PatchDisposition) + UserProfile, PatchDisposition, ExtendedProvide) from . import tasks, utils @@ -1818,3 +1818,115 @@ class BranchCompareView(FormView): return context + +class RecipeDependenciesView(FormView): + form_class = RecipeDependenciesForm + + def get_recipes(self, layerbranch, exclude_layer_ids, crosslayer): + class RecipeResult: + def __init__(self, id, pn, short_desc, license): + self.id = id + self.pn = pn + self.short_desc = short_desc + self.license = license + self.deps = [] + class RecipeDependencyResult: + def __init__(self, id, depname, pn, pv, license, layer, dynamic): + self.id = id + self.depname = depname + self.pn = pn + self.pv = pv + self.license = license + self.layer = layer + self.dynamic = dynamic + + recipes = Recipe.objects.filter(layerbranch=layerbranch) + + layerprovides = [] + if crosslayer: + layerprovides = list(ExtendedProvide.objects.filter(recipes__layerbranch=layerbranch).values_list('name', flat=True)) + + branch = layerbranch.branch + + def process(resultobj, depname, dynamic): + if crosslayer and depname in layerprovides: + return + eprovides = ExtendedProvide.objects.filter(name=depname) + if eprovides: + for eprovide in eprovides: + deprecipes = eprovide.recipes.filter(layerbranch__branch=branch).values('id', 'pn', 'pv', 'license', 'layerbranch__layer__name').order_by('-layerbranch__layer__index_preference', 'layerbranch', 'pn') + if exclude_layer_ids: + deprecipes = deprecipes.exclude(layerbranch__layer__in=exclude_layer_ids) + for deprecipe in deprecipes: + resultobj.deps.append(RecipeDependencyResult(deprecipe['id'], + depname, + deprecipe['pn'], + deprecipe['pv'], + deprecipe['license'], + deprecipe['layerbranch__layer__name'], + dynamic)) + if not resultobj.deps: + resultobj.deps.append(RecipeDependencyResult(-1, + depname, + depname, + '', + '', + '', + dynamic)) + + outrecipes = [] + for recipe in recipes: + res = RecipeResult(recipe.id, recipe.pn, recipe.short_desc, recipe.license) + for rdepname in recipe.staticbuilddep_set.values_list('name', flat=True).order_by('name'): + process(res, rdepname, False) + for rdepname in recipe.dynamicbuilddep_set.values_list('name', flat=True).order_by('name'): + process(res, rdepname, True) + outrecipes.append(res) + + return outrecipes + + def form_valid(self, form): + return HttpResponseRedirect(reverse_lazy('recipe_deps', args=(form.cleaned_data['branch'].name))) + + def get_initial(self): + initial = super(RecipeDependenciesView, self).get_initial() + branch_id = self.request.GET.get('branch', None) + if branch_id is not None: + initial['branch'] = get_object_or_404(Branch, id=branch_id) + layer_id = self.request.GET.get('layer', None) + if layer_id is not None: + initial['layer'] = get_object_or_404(LayerItem, id=layer_id) + initial['excludelayers'] = self.request.GET.get('excludelayers', '') + initial['crosslayer'] = self.request.GET.get('crosslayer', False) + return initial + + def get_context_data(self, **kwargs): + context = super(RecipeDependenciesView, self).get_context_data(**kwargs) + branch_id = self.request.GET.get('branch', None) + layer_id = self.request.GET.get('layer', None) + exclude_layer_ids = self.request.GET.get('excludelayers', '') + if exclude_layer_ids: + exclude_layer_ids = exclude_layer_ids.split(',') + branch = None + if branch_id is not None: + branch = get_object_or_404(Branch, id=branch_id) + context['branch'] = branch + layer = None + if layer_id is not None: + layer = get_object_or_404(LayerItem, id=layer_id) + context['layer'] = layer + crosslayer = self.request.GET.get('crosslayer', False) + context['crosslayer'] = crosslayer + layerbranch = None + if layer: + layerbranch = layer.get_layerbranch(branch.name) + if layerbranch: + context['recipes'] = self.get_recipes(layerbranch, exclude_layer_ids, crosslayer) + context['this_url_name'] = resolve(self.request.path_info).url_name + context['layers'] = LayerItem.objects.filter(status__in=['P', 'X']).order_by('name') + context['excludelayers'] = exclude_layer_ids + layerlist = dict(context['layers'].values_list('id', 'name')) + context['excludelayers_text'] = ', '.join([layerlist[int(i)] for i in exclude_layer_ids]) + + return context + diff --git a/templates/base.html b/templates/base.html index 126784d..64c721e 100644 --- a/templates/base.html +++ b/templates/base.html @@ -88,6 +88,7 @@
  • Updates
  • Statistics
  • Branch Comparison
    • +
  • Recipe Dependencies
    • {% if rrs_enabled %}
  • Recipe Maintenance
    • {% endif %} diff --git a/templates/layerindex/recipedeps.html b/templates/layerindex/recipedeps.html new file mode 100644 index 0000000..26e19e9 --- /dev/null +++ b/templates/layerindex/recipedeps.html @@ -0,0 +1,209 @@ +{% extends "base.html" %} +{% load i18n %} +{% load static %} + +{% comment %} + + layerindex-web - recipe dependencies page template + + Copyright (C) 2019 Intel Corporation + Licensed under the MIT license, see COPYING.MIT for details + +{% endcomment %} + + + + +{% block content %} +{% autoescape on %} + +

    Recipe dependencies

    + +
    +
    + +
    + {% for hidden in form.hidden_fields %} + {{ hidden }} + {% endfor %} + {% for field in form.visible_fields %} + {% if field.name in form.errors %} +
    + {{ field.errors }} + {% else %} +
    + {% endif %} + {% if field.name == "crosslayer" %} +
    + + +
    + {% else %} +
    + {{ field.label_tag }} +
    +
    + {{ field }} +
    + {% endif %} +

    + {{ field.help_text|safe }} +

    +
    + {% endfor %} + + + +
    + Exclude layers {{ excludelayers|length }} +
    + + + +
    + +
    +
    +{% if recipes %} + + + + + + + + + + + + + + {% for recipe in recipes %} + {% with len=recipe.deps|length %} + {% for dep in recipe.deps %} + + {% if forloop.first %} + + + {% endif %} + + + + + + {% endfor %} + {% endwith %} + {% endfor %} + +
    RecipeLicenseDependencyVersion - {{ branch }}LicenseLayer
    {{ recipe.pn }}{{ recipe.license }}{% if dep.pn != dep.depname %}{{ dep.depname }}: {% endif %}{% if dep.id > -1 %}{% endif %}{{ dep.pn }}{% if dep.id > -1 %}{% endif %}{% if dep.dynamic %} optional{% endif %}{{ dep.pv }}{{ dep.license }}{{ dep.layer }}
    +{% elif branch %} +

    No matching recipes in database.

    +{% endif %} +
    +
    + + + Plain text + + + +{% endautoescape %} + +{% endblock %} + + +{% block scripts %} + +{% endblock %}