Add recipe dependencies tool

Add an extra tool that lets you view all of the recipe dependencies in
a layer. There is also a mode that shows only cross-layer dependencies,
which can be useful to find dependencies on recipes in other layers
that aren't declared in the layer's dependencies (or conversely where a
layer dependency is no longer necessary).

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

layerindex/admin.py

@@ -234,6 +234,7 @@ admin.site.register(LayerUpdate, LayerUpdateAdmin)
 admin.site.register(PackageConfig, PackageConfigAdmin)
 admin.site.register(StaticBuildDep, StaticBuildDepAdmin)
 admin.site.register(DynamicBuildDep, DynamicBuildDepAdmin)
+admin.site.register(ExtendedProvide)
 admin.site.register(Source, SourceAdmin)
 admin.site.register(Recipe, RecipeAdmin)
 admin.site.register(RecipeFileDependency)

layerindex/forms.py

@@ -373,3 +373,16 @@ class BranchComparisonForm(StyledForm):
         if cleaned_data['from_branch'] == cleaned_data['to_branch']:
             raise forms.ValidationError({'to_branch': 'From and to branches cannot be the same'})
         return cleaned_data
+
+
+class RecipeDependenciesForm(StyledForm):
+    branch = forms.ModelChoiceField(label='Branch', queryset=Branch.objects.none())
+    layer = forms.ModelChoiceField(queryset=LayerItem.objects.filter(comparison=False).filter(status__in=['P', 'X']).order_by('name'), required=True)
+    crosslayer = forms.BooleanField(required=False)
+    excludelayers = forms.CharField(widget=forms.HiddenInput())
+
+    def __init__(self, *args, request=None, **kwargs):
+        super(RecipeDependenciesForm, self).__init__(*args, **kwargs)
+        qs = Branch.objects.filter(comparison=False, hidden=False).order_by('sort_priority', 'name')
+        self.fields['branch'].queryset = qs
+        self.request = request

layerindex/migrations/0044_extendedprovides.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-11-05 22:51
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+def populate_extended_provides(apps, schema_editor):
+    Branch = apps.get_model('layerindex', 'Branch')
+    LayerBranch = apps.get_model('layerindex', 'LayerBranch')
+    Recipe = apps.get_model('layerindex', 'Recipe')
+    ExtendedProvide = apps.get_model('layerindex', 'ExtendedProvide')
+
+    for branch in Branch.objects.filter(comparison=False):
+        for layerbranch in LayerBranch.objects.filter(branch=branch):
+            for recipe in Recipe.objects.filter(layerbranch=layerbranch):
+                provides = recipe.provides.split()
+                for extend in recipe.bbclassextend.split():
+                    if extend == 'native':
+                        provides.append('%s-native' % recipe.pn)
+                    elif extend == 'nativesdk':
+                        provides.append('nativesdk-%s' % recipe.pn)
+                for provide in provides:
+                    provides, created = ExtendedProvide.objects.get_or_create(name=provide)
+                    if created:
+                        provides.save()
+                    provides.recipes.add(recipe)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('layerindex', '0043_recipe_srcrev'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ExtendedProvide',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255, unique=True)),
+                ('recipes', models.ManyToManyField(to='layerindex.Recipe')),
+            ],
+        ),
+        migrations.RunPython(populate_extended_provides, reverse_code=migrations.RunPython.noop),
+    ]

layerindex/models.py

@@ -919,3 +919,11 @@ class PatchDisposition(models.Model):
 
     def __str__(self):
         return '%s - %s' % (self.patch, self.get_disposition_display())
+
+
+class ExtendedProvide(models.Model):
+    recipes = models.ManyToManyField(Recipe)
+    name = models.CharField(max_length=255, unique=True)
+
+    def __str__(self):
+        return self.name

layerindex/recipeparse.py

@@ -195,3 +195,18 @@ def handle_recipe_depends(recipe, depends, packageconfig_opts):
                 dynamic_build_dependency.package_configs.add(package_config)
                 dynamic_build_dependency.recipes.add(recipe)
 
+def handle_recipe_provides(recipe):
+    from layerindex.models import ExtendedProvide
+
+    recipe.extendedprovide_set.clear()
+    provides = recipe.provides.split()
+    for extend in recipe.bbclassextend.split():
+        if extend == 'native':
+            provides.append('%s-native' % recipe.pn)
+        elif extend == 'nativesdk':
+            provides.append('nativesdk-%s' % recipe.pn)
+    for provide in provides:
+        provides, created = ExtendedProvide.objects.get_or_create(name=provide)
+        if created:
+            provides.save()
+        provides.recipes.add(recipe)

layerindex/update_layer.py

@@ -150,6 +150,8 @@ def update_recipe_file(tinfoil, data, path, recipe, layerdir_start, repodir, sto
 
         recipeparse.handle_recipe_depends(recipe, envdata.getVar('DEPENDS', True) or '', envdata.getVarFlags('PACKAGECONFIG'))
 
+        recipeparse.handle_recipe_provides(recipe)
+
         if not skip_patches:
             # Handle patches
             collect_patches(recipe, envdata, layerdir_start, stop_on_error)

layerindex/urls.py

@@ -15,7 +15,7 @@ from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDeta
     ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView, LayerUpdateDetailView, UpdateListView, \
     UpdateDetailView, StatsView, publish_view, LayerCheckListView, BBClassCheckListView, TaskStatusView, \
     ComparisonRecipeSelectView, ComparisonRecipeSelectDetailView, task_log_view, task_stop_view, email_test_view, \
-    BranchCompareView
+    BranchCompareView, RecipeDependenciesView
 from layerindex.models import LayerItem, Recipe, RecipeChangeset
 from rest_framework import routers
 from . import restviews
@@ -195,6 +195,10 @@ urlpatterns = [
             content_type='text/plain',
             template_name='layerindex/branchcompare_plain.txt'),
         name='branch_comparison_plain'),
+    url(r'^recipe_deps/$',
+        RecipeDependenciesView.as_view(
+            template_name='layerindex/recipedeps.html'),
+        name='recipe_deps'),
     url(r'^ajax/layerchecklist/(?P<branch>[-.\w]+)/$',
         LayerCheckListView.as_view(
             template_name='layerindex/layerchecklist.html'),

layerindex/views.py

@@ -48,14 +48,14 @@ from layerindex.forms import (AdvancedRecipeSearchForm, BulkChangeEditFormSet,
                               EditNoteForm, EditProfileForm,
                               LayerMaintainerFormSet, RecipeChangesetForm,
                               PatchDispositionForm, PatchDispositionFormSet,
-                              BranchComparisonForm)
+                              BranchComparisonForm, RecipeDependenciesForm)
 from layerindex.models import (BBAppend, BBClass, Branch, ClassicRecipe,
                                Distro, DynamicBuildDep, IncFile, LayerBranch,
                                LayerDependency, LayerItem, LayerMaintainer,
                                LayerNote, LayerUpdate, Machine, Patch, Recipe,
                                RecipeChange, RecipeChangeset, Source, StaticBuildDep,
                                Update, SecurityQuestion, SecurityQuestionAnswer,
-                               UserProfile, PatchDisposition)
+                               UserProfile, PatchDisposition, ExtendedProvide)
 
 
 from . import tasks, utils
@@ -1818,3 +1818,115 @@ class BranchCompareView(FormView):
 
         return context
 
+
+class RecipeDependenciesView(FormView):
+    form_class = RecipeDependenciesForm
+
+    def get_recipes(self, layerbranch, exclude_layer_ids, crosslayer):
+        class RecipeResult:
+            def __init__(self, id, pn, short_desc, license):
+                self.id = id
+                self.pn = pn
+                self.short_desc = short_desc
+                self.license = license
+                self.deps = []
+        class RecipeDependencyResult:
+            def __init__(self, id, depname, pn, pv, license, layer, dynamic):
+                self.id = id
+                self.depname = depname
+                self.pn = pn
+                self.pv = pv
+                self.license = license
+                self.layer = layer
+                self.dynamic = dynamic
+
+        recipes = Recipe.objects.filter(layerbranch=layerbranch)
+
+        layerprovides = []
+        if crosslayer:
+            layerprovides = list(ExtendedProvide.objects.filter(recipes__layerbranch=layerbranch).values_list('name', flat=True))
+
+        branch = layerbranch.branch
+
+        def process(resultobj, depname, dynamic):
+            if crosslayer and depname in layerprovides:
+                return
+            eprovides = ExtendedProvide.objects.filter(name=depname)
+            if eprovides:
+                for eprovide in eprovides:
+                    deprecipes = eprovide.recipes.filter(layerbranch__branch=branch).values('id', 'pn', 'pv', 'license', 'layerbranch__layer__name').order_by('-layerbranch__layer__index_preference', 'layerbranch', 'pn')
+                    if exclude_layer_ids:
+                        deprecipes = deprecipes.exclude(layerbranch__layer__in=exclude_layer_ids)
+                    for deprecipe in deprecipes:
+                        resultobj.deps.append(RecipeDependencyResult(deprecipe['id'],
+                                                            depname,
+                                                            deprecipe['pn'],
+                                                            deprecipe['pv'],
+                                                            deprecipe['license'],
+                                                            deprecipe['layerbranch__layer__name'],
+                                                            dynamic))
+            if not resultobj.deps:
+                resultobj.deps.append(RecipeDependencyResult(-1,
+                                                    depname,
+                                                    depname,
+                                                    '',
+                                                    '',
+                                                    '',
+                                                    dynamic))
+
+        outrecipes = []
+        for recipe in recipes:
+            res = RecipeResult(recipe.id, recipe.pn, recipe.short_desc, recipe.license)
+            for rdepname in recipe.staticbuilddep_set.values_list('name', flat=True).order_by('name'):
+                process(res, rdepname, False)
+            for rdepname in recipe.dynamicbuilddep_set.values_list('name', flat=True).order_by('name'):
+                process(res, rdepname, True)
+            outrecipes.append(res)
+
+        return outrecipes
+
+    def form_valid(self, form):
+        return HttpResponseRedirect(reverse_lazy('recipe_deps', args=(form.cleaned_data['branch'].name)))
+
+    def get_initial(self):
+        initial = super(RecipeDependenciesView, self).get_initial()
+        branch_id = self.request.GET.get('branch', None)
+        if branch_id is not None:
+            initial['branch'] = get_object_or_404(Branch, id=branch_id)
+        layer_id = self.request.GET.get('layer', None)
+        if layer_id is not None:
+            initial['layer'] = get_object_or_404(LayerItem, id=layer_id)
+        initial['excludelayers'] = self.request.GET.get('excludelayers', '')
+        initial['crosslayer'] = self.request.GET.get('crosslayer', False)
+        return initial
+
+    def get_context_data(self, **kwargs):
+        context = super(RecipeDependenciesView, self).get_context_data(**kwargs)
+        branch_id = self.request.GET.get('branch', None)
+        layer_id = self.request.GET.get('layer', None)
+        exclude_layer_ids = self.request.GET.get('excludelayers', '')
+        if exclude_layer_ids:
+            exclude_layer_ids = exclude_layer_ids.split(',')
+        branch = None
+        if branch_id is not None:
+            branch = get_object_or_404(Branch, id=branch_id)
+        context['branch'] = branch
+        layer = None
+        if layer_id is not None:
+            layer = get_object_or_404(LayerItem, id=layer_id)
+        context['layer'] = layer
+        crosslayer = self.request.GET.get('crosslayer', False)
+        context['crosslayer'] = crosslayer
+        layerbranch = None
+        if layer:
+            layerbranch = layer.get_layerbranch(branch.name)
+        if layerbranch:
+            context['recipes'] = self.get_recipes(layerbranch, exclude_layer_ids, crosslayer)
+        context['this_url_name'] = resolve(self.request.path_info).url_name
+        context['layers'] = LayerItem.objects.filter(status__in=['P', 'X']).order_by('name')
+        context['excludelayers'] = exclude_layer_ids
+        layerlist = dict(context['layers'].values_list('id', 'name'))
+        context['excludelayers_text'] = ', '.join([layerlist[int(i)] for i in exclude_layer_ids])
+
+        return context
+

templates/base.html

@@ -88,6 +88,7 @@
                             <li><a href="{% url 'update_list' %}">Updates</a></li>
                             <li><a href="{% url 'stats' %}">Statistics</a></li>
                             <li><a href="{% url 'branch_comparison' %}">Branch Comparison</a></li>
+                            <li><a href="{% url 'recipe_deps' %}">Recipe Dependencies</a></li>
                             {% if rrs_enabled %}
                             <li><a href="{% url 'rrs_frontpage' %}">Recipe Maintenance</a></li>
                             {% endif %}

templates/layerindex/recipedeps.html

@@ -0,0 +1,209 @@
+{% extends "base.html" %}
+{% load i18n %}
+{% load static %}
+
+{% comment %}
+
+  layerindex-web - recipe dependencies page template
+
+  Copyright (C) 2019 Intel Corporation
+  Licensed under the MIT license, see COPYING.MIT for details
+
+{% endcomment %}
+
+
+<!--
+{% block title_append %} - recipe dependencies{% endblock %}
+-->
+
+{% block content %}
+{% autoescape on %}
+
+    <h2>Recipe dependencies</h2>
+
+        <div class="row">
+            <div class="col-md-5">
+
+        <form method="GET">
+        {% for hidden in form.hidden_fields %}
+            {{ hidden }}
+        {% endfor %}
+    {% for field in form.visible_fields %}
+        {% if field.name in form.errors %}
+        <div class="form-group alert alert-danger">
+        {{ field.errors }}
+        {% else %}
+        <div class="form-group">
+        {% endif %}
+            {% if field.name == "crosslayer" %}
+            <div class="controls">
+                <input type="checkbox" name="crosslayer" id="id_crosslayer" {% if crosslayer %} checked{% endif %} />
+                <label for="id_crosslayer">Cross-layer deps only</label></td>
+            </div>
+            {% else %}
+            <div class="control-label {% if field.required %}requiredlabel{% endif %}">
+                {{ field.label_tag }}
+            </div>
+            <div class="controls">
+                {{ field }}
+            </div>
+            {% endif %}
+        <p>
+        {{ field.help_text|safe }}
+        </p>
+        </div>
+    {% endfor %}
+
+            <div id="layerDialog" class="modal fade" tabindex="-1" role="dialog" aria-labelledby="layerDialogLabel">
+                <div class="modal-dialog" role="document">
+                    <div class="modal-content">
+                        <div class="modal-header">
+                            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+                            <h3 id="layerDialogLabel">Select layers to exclude for dependencies</h3>
+                        </div>
+                            <div class="modal-body">
+                                <div class="form-group has-feedback has-clear">
+                                    <input type="text" class="form-control" id="layersearchtext" placeholder="search layers">
+                                    <a class="glyphicon glyphicon-remove-sign form-control-feedback form-control-clear" id="layersearchclear" style="pointer-events: auto; text-decoration: none;cursor: pointer;"></a>
+                                </div>
+                                <div class="scrolling">
+                                    <table class="layerstable"><tbody>
+                                        {% for layer in layers %}
+                                            <tr>
+                                                <td class="checkboxtd"><input
+                                                    type="checkbox"
+                                                    class="filterlayercheckbox"
+                                                    value="{{ layer.id }}" id="id_excludelayercheckbox_{{layer.id}}"
+                                                    {% if excludelayers and layer.id in excludelayers %}
+                                                        checked
+                                                    {% endif %}
+                                                    />
+                                                </td>
+                                                <td><label for="id_excludelayercheckbox_{{layer.id}}">{{ layer.name }}</label></td>
+                                            </tr>
+                                        {% endfor %}
+                                    </tbody></table>
+                                </div>
+                                <div class="buttonblock">
+                                <button type="button" class="btn btn-default buttonblock-btn" id="id_select_none">Clear selections</button>
+                                </div>
+                            </div>
+                            <div class="modal-footer">
+                                <button type="button" class="btn btn-primary" id="id_layerdialog_ok" data-dismiss="modal">Exclude</button>
+                                <button type="button" class="btn btn-default" id="id_cancel" data-dismiss="modal">Cancel</button>
+                            </div>
+                    </div><!-- /.modal-content -->
+                </div><!-- /.modal-dialog -->
+            </div>
+
+        <div class="form-group">
+            <a href="#layerDialog" role="button" id="id_select_layers" class="btn btn-default nav-spacer" data-toggle="modal">Exclude layers <span class="badge badge-info" id="id_excludelayers_count">{{ excludelayers|length }}</span></a>
+        </div>
+
+            <button type="submit" class="btn btn-primary">Show</button>
+        </form>
+        <br>
+
+            </div>
+            <div class="col-md-12">
+{% if recipes %}
+                <table class="table table-striped table-bordered recipestable">
+                    <thead>
+                        <tr>
+                            <th>Recipe</th>
+                            <th>License</th>
+                            <th>Dependency</th>
+                            <th>Version - {{ branch }}</th>
+                            <th>License</th>
+                            <th>Layer</th>
+                        </tr>
+                    </thead>
+
+                    <tbody>
+                        {% for recipe in recipes %}
+                            {% with len=recipe.deps|length %}
+                            {% for dep in recipe.deps %}
+                            <tr>
+                                {% if forloop.first %}
+                                <td rowspan="{{ len }}"><a href="{% url 'recipe' recipe.id %}">{{ recipe.pn }}</a></td>
+                                <td rowspan="{{ len }}">{{ recipe.license }}</td>
+                                {% endif %}
+                                <td>{% if dep.pn != dep.depname %}{{ dep.depname }}: {% endif %}{% if dep.id > -1 %}<a href="{% url 'recipe' dep.id %}">{% endif %}{{ dep.pn }}{% if dep.id > -1 %}</a>{% endif %}{% if dep.dynamic %} <span class="label label-default">optional</span>{% endif %}</td>
+                                <td>{{ dep.pv }}</td>
+                                <td>{{ dep.license }}</td>
+                                <td>{{ dep.layer }}</td>
+                            </tr>
+                            {% endfor %}
+                            {% endwith %}
+                        {% endfor %}
+                    </tbody>
+                </table>
+{% elif branch %}
+    <p>No matching recipes in database.</p>
+{% endif %}
+            </div>
+        </div>
+
+    <span class="pull-right">
+    <a class="btn btn-default" href="{% url 'branch_comparison_plain' %}?{{ request.GET.urlencode }}"><i class="glyphicon glyphicon-file"></i> Plain text</a>
+    </span>
+
+
+{% endautoescape %}
+
+{% endblock %}
+
+
+{% block scripts %}
+<script>
+    $(document).ready(function() {
+        firstfield = $("#filter-form input:text").first()
+        if( ! firstfield.val() )
+            firstfield.focus()
+    });
+    $('#id_select_none').click(function (e) {
+        $('.layerstable').find('tr:visible').find('.filterlayercheckbox').prop('checked', false);
+    });
+
+    function clearLayerSearch() {
+        $("#layersearchtext").val('');
+        $(".layerstable > tbody > tr").show();
+    }
+
+    update_selected_layer_display = function() {
+        //layernames = [];
+        layerids = [];
+        $('.filterlayercheckbox:checked').each(function() {
+            //layernames.push($("label[for="+$(this).attr('id')+"]").html());
+            layerids.push($(this).attr('value'))
+        });
+        $('#id_excludelayers').val(layerids)
+        $('#id_excludelayers_count').html(layerids.length)
+    }
+    select_layer_checkboxes = function() {
+        $('.filterlayercheckbox').prop('checked', false);
+        selectedlayers = $('#id_excludelayers').val().split(',');
+        for(i in selectedlayers) {
+            $('#id_excludelayercheckbox_' + selectedlayers[i]).prop('checked', true);
+        }
+    }
+
+    $('#id_layerdialog_ok').click(function (e) {
+        update_selected_layer_display()
+    });
+    $("#layersearchtext").on("input", function() {
+        var value = $(this).val().toLowerCase();
+        $(".layerstable > tbody > tr").filter(function() {
+            $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1)
+        });
+    });
+    $("#layersearchclear").click(function(){
+        clearLayerSearch();
+        $("#layersearchtext").focus();
+    });
+    $('#id_select_layers').click(function (e) {
+        clearLayerSearch();
+        select_layer_checkboxes();
+    })
+</script>
+{% endblock %}