diff --git a/README b/README index cd54b60..cd6d6a6 100644 --- a/README +++ b/README @@ -15,16 +15,12 @@ In order to make use of this application you will need: * A database supported by Django (SQLite, MySQL, etc.). Django takes care of creating the database itself, you just need to ensure that the database server (if not using SQLite) is configured and running. -* On the machine that will run the update script (which does not have to - be the same machine as the web server, however it does still have to - have Django installed and have access to the database used by the web +* On the machine that will run the backend update script (which does not + have to be the same machine as the web server, however it does still + have to have Django installed, have the same or similar configuration + in settings.py and have access to the database used by the web application): * Python 2.6 or Python 2.7 - * A copy of BitBake and OE-Core (or Poky, which includes both) - - the "danny" release or newer is required. It does not need to be - configured specially nor do all of the normal pre-requisites need to - be installed (it is only used for parsing recipes, not actual - building). * GitPython (python-git) version 0.3.1 or later * django-registration * django-reversion @@ -43,31 +39,45 @@ Setup instructions: python manage.py syncdb + You should answer "yes" when asked to create an admin account. + 3. You can test the web application locally by setting DEBUG = True in settings.py and running the following: python manage.py runserver - Then visit http://127.0.0.1:8000/layerindex/ with your browser. This - should only be used for testing - for production you need to use a - proper web server and have DEBUG set to False. + Then visit http://127.0.0.1:8000/layerindex/ with your browser. As + with all Django applications there is an admin interface available + at http://127.0.0.1:8000/admin/ also. -4. You may wish to customise layerindex/about.html to suit your + NOTE: This local server should only be used for testing - for + production you need to use a proper web server and have DEBUG set + to False. + +4. You'll need to add at least the openembedded-core layer to the + database (or some equivalent for use in basic BitBake configuration; + if it's not called "openembedded-core" then you'll need to set + CORE_LAYER_NAME in settings.py to match.) You can use the + "Submit Layer" feature within the interface itself (easiest) or the + admin interface to do this. + +5. You may wish to customise layerindex/about.html to suit your installation. Usage ----- -On a regular basis you need to run the update script within an -environment set up for OE-Core build: +On a regular basis you need to run the update script: -$ cd path/to/oe-core -$ . ./oe-init-build-env -$ path/to/layerindex/update.py +path/to/layerindex/update.py This will fetch all of the layer repositories, analyse their contents -and update the database with the results. +and update the database with the results. Note that if you set up more +than just the master branch in the database, you will need to run the +script once for each branch using -b (or --branch) to specify the +branch name. Run the script with --help for further information on +available options. Maintenance diff --git a/base.html b/base.html index ac9beef..75871fe 100644 --- a/base.html +++ b/base.html @@ -30,6 +30,25 @@
OpenEmbedded metadata index +
+ {% if not layerbranch %} +
+
+ This layer does not exist for branch {{ current_branch }}. +
+
+ {% endif %}
{% for note in layeritem.layernote_set.all %}
@@ -95,24 +102,25 @@ {% if layeritem.vcs_subdir %}

Subdirectory

{{ layeritem.vcs_subdir }} - {% if layeritem.tree_url %} + {% if layerbranch.tree_url %} - web subdirectory + web subdirectory {% endif %}

{% endif %} - {% if layeritem.vcs_last_commit %} + {% if layerbranch.vcs_last_commit %}

- Last commit: {{ layeritem.vcs_last_commit|timesince }} ago + Last commit: {{ layerbranch.vcs_last_commit|timesince }} ago

{% endif %} -

{% if layeritem.active_maintainers|length = 1 %}Maintainer{% else %}Maintainers{% endif %}

+ {% if layerbranch.active_maintainers.count > 0 %} +

{% if layerbranch.active_maintainers|length = 1 %}Maintainer{% else %}Maintainers{% endif %}

    - {% for maintainer in layeritem.active_maintainers %} + {% for maintainer in layerbranch.active_maintainers %}
  • {{ maintainer.name }} {% if maintainer.responsibility %} @@ -125,16 +133,17 @@
    • {% endfor %}
+ {% endif %}
- {% if layeritem.dependencies_set.all %} + {% if layerbranch.dependencies_set.count > 0 %}

Dependencies

The {{ layeritem.name }} layer depends upon:

    - {% for dep in layeritem.dependencies_set.all %} + {% for dep in layerbranch.dependencies_set.all %}
  • {{ dep.dependency.name }}
    • {% endfor %}
@@ -145,11 +154,11 @@
- {% if layeritem.status = 'N' %} + {% if layeritem.status = 'N' and layerbranch %}
{% if layeritem.vcs_web_file_base_url %} - test file link + test file link {% else %} @@ -158,7 +167,7 @@ {% endif %} {% if layeritem.vcs_web_tree_base_url %} - test tree link + test tree link {% else %} @@ -167,7 +176,7 @@ {% endif %}
{% else %} - {% if layeritem.machine_set.count > 0 %} + {% if layerbranch.machine_set.count > 0 %}
@@ -178,7 +187,7 @@ - {% for machine in layeritem.machine_set.all %} + {% for machine in layerbranch.machine_set.all %} @@ -190,6 +199,7 @@ {% endif %} + {% if layerbranch.recipe_set.count > 0 %}
@@ -217,7 +227,7 @@
- {% for recipe in layeritem.sorted_recipes %} + {% for recipe in layerbranch.sorted_recipes %} @@ -229,6 +239,7 @@ {% endif %} + {% endif %} {% endautoescape %} diff --git a/layerindex/editlayer.html b/layerindex/editlayer.html index 5f6b691..9c65c84 100644 --- a/layerindex/editlayer.html +++ b/layerindex/editlayer.html @@ -180,35 +180,35 @@ if( repoval.startsWith('git://git.openembedded.org/') ) { reponame = repoval.replace(/^.*\//, '') $('#id_vcs_web_url').val('http://cgit.openembedded.org/cgit.cgi/' + reponame) - $('#id_vcs_web_tree_base_url').val('http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/') - $('#id_vcs_web_file_base_url').val('http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/') + $('#id_vcs_web_tree_base_url').val('http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%') + $('#id_vcs_web_file_base_url').val('http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%') } else if( repoval.indexOf('git.yoctoproject.org/') > -1 ) { reponame = repoval.replace(/^.*\//, '') $('#id_vcs_web_url').val('http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame) - $('#id_vcs_web_tree_base_url').val('http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/') - $('#id_vcs_web_file_base_url').val('http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/') + $('#id_vcs_web_tree_base_url').val('http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%') + $('#id_vcs_web_file_base_url').val('http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%') } else if( repoval.indexOf('github.com/') > -1 ) { reponame = repoval.replace(/^.*github.com\//, '') reponame = reponame.replace(/.git$/, '') $('#id_vcs_web_url').val('http://github.com/' + reponame) - $('#id_vcs_web_tree_base_url').val('http://github.com/' + reponame + '/tree/master/') - $('#id_vcs_web_file_base_url').val('http://github.com/' + reponame + '/blob/master/') + $('#id_vcs_web_tree_base_url').val('http://github.com/' + reponame + '/tree/%branch%/') + $('#id_vcs_web_file_base_url').val('http://github.com/' + reponame + '/blob/%branch%/') } else if( repoval.indexOf('gitorious.org/') > -1 ) { reponame = repoval.replace(/^.*gitorious.org\//, '') reponame = reponame.replace(/.git$/, '') $('#id_vcs_web_url').val('http://gitorious.org/' + reponame) - $('#id_vcs_web_tree_base_url').val('http://gitorious.org/' + reponame + '/trees/master/') - $('#id_vcs_web_file_base_url').val('http://gitorious.org/' + reponame + '/blobs/master/') + $('#id_vcs_web_tree_base_url').val('http://gitorious.org/' + reponame + '/trees/%branch%/') + $('#id_vcs_web_file_base_url').val('http://gitorious.org/' + reponame + '/blobs/%branch%/') } else if( repoval.indexOf('bitbucket.org/') > -1 ) { reponame = repoval.replace(/^.*bitbucket.org\//, '') reponame = reponame.replace(/.git$/, '') $('#id_vcs_web_url').val('http://bitbucket.org/' + reponame) - $('#id_vcs_web_tree_base_url').val('http://bitbucket.org/' + reponame + '/src/master/%path%?at=master') - $('#id_vcs_web_file_base_url').val('http://bitbucket.org/' + reponame + '/src/master/%path%?at=master') + $('#id_vcs_web_tree_base_url').val('http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%') + $('#id_vcs_web_file_base_url').val('http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%') } }; diff --git a/layerindex/fixtures/initial_data.json b/layerindex/fixtures/initial_data.json new file mode 100644 index 0000000..36072a5 --- /dev/null +++ b/layerindex/fixtures/initial_data.json @@ -0,0 +1,10 @@ +[ + { + "model": "layerindex.branch", + "pk": 1, + "fields": { + "name": "master", + "bitbake_branch": "master" + } + } +] \ No newline at end of file diff --git a/layerindex/forms.py b/layerindex/forms.py index bc1134e..5932647 100644 --- a/layerindex/forms.py +++ b/layerindex/forms.py @@ -4,7 +4,7 @@ # # Licensed under the MIT license, see COPYING.MIT for details -from layerindex.models import LayerItem, LayerMaintainer, LayerNote +from layerindex.models import LayerItem, LayerBranch, LayerMaintainer, LayerNote from django import forms from django.core.validators import URLValidator, RegexValidator, email_re from django.forms.models import inlineformset_factory @@ -42,7 +42,7 @@ class BaseLayerMaintainerFormSet(forms.models.BaseInlineFormSet): f.required = True return f -LayerMaintainerFormSet = inlineformset_factory(LayerItem, LayerMaintainer, form=LayerMaintainerForm, formset=BaseLayerMaintainerFormSet, can_delete=False, extra=10, max_num=10) +LayerMaintainerFormSet = inlineformset_factory(LayerBranch, LayerMaintainer, form=LayerMaintainerForm, formset=BaseLayerMaintainerFormSet, can_delete=False, extra=10, max_num=10) class EditLayerForm(forms.ModelForm): # Additional form fields @@ -53,10 +53,11 @@ class EditLayerForm(forms.ModelForm): model = LayerItem fields = ('name', 'layer_type', 'summary', 'description', 'vcs_url', 'vcs_subdir', 'vcs_web_url', 'vcs_web_tree_base_url', 'vcs_web_file_base_url', 'usage_url', 'mailing_list_url') - def __init__(self, user, *args, **kwargs): + def __init__(self, user, layerbranch, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) + self.layerbranch = layerbranch if self.instance.pk: - self.fields['deps'].initial = [d.dependency.pk for d in self.instance.dependencies_set.all()] + self.fields['deps'].initial = [d.dependency.pk for d in self.layerbranch.dependencies_set.all()] del self.fields['captcha'] else: self.fields['deps'].initial = [l.pk for l in LayerItem.objects.filter(name='openembedded-core')] diff --git a/layerindex/index.html b/layerindex/index.html index d6b192d..461252a 100644 --- a/layerindex/index.html +++ b/layerindex/index.html @@ -19,7 +19,7 @@ {% block content %} {% autoescape on %} -{% if layer_list %} +{% if layerbranch_list %}
@@ -68,22 +68,22 @@
- {% for layeritem in layer_list %} - - - - + {% for layerbranch in layerbranch_list %} + + + + - + {% endfor %} diff --git a/layerindex/models.py b/layerindex/models.py index ee5a063..a32a0b0 100644 --- a/layerindex/models.py +++ b/layerindex/models.py @@ -9,6 +9,18 @@ from datetime import datetime from django.contrib.auth.models import User from django.core.urlresolvers import reverse import os.path +import re + +class Branch(models.Model): + name = models.CharField(max_length=50) + bitbake_branch = models.CharField(max_length=50) + + class Meta: + verbose_name_plural = "Branches" + + def __unicode__(self): + return self.name + class LayerItem(models.Model): LAYER_STATUS_CHOICES = ( @@ -27,9 +39,6 @@ class LayerItem(models.Model): layer_type = models.CharField(max_length=1, choices=LAYER_TYPE_CHOICES) summary = models.CharField(max_length=200, help_text='One-line description of the layer') description = models.TextField() - vcs_last_fetch = models.DateTimeField('Last successful fetch', blank=True, null=True) - vcs_last_rev = models.CharField('Last revision fetched', max_length=80, blank=True) - vcs_last_commit = models.DateTimeField('Last commit date', blank=True, null=True) vcs_subdir = models.CharField('Repository subdirectory', max_length=40, blank=True, help_text='Subdirectory within the repository where the layer is located, if not in the root (usually only used if the repository contains more than one layer)') vcs_url = models.CharField('Repository URL', max_length=200, help_text='Fetch/clone URL of the repository') vcs_web_url = models.URLField('Repository web interface URL', blank=True, help_text='URL of the web interface for browsing the repository, if any') @@ -47,40 +56,21 @@ class LayerItem(models.Model): def change_status(self, newstatus, username): self.status = newstatus - def _handle_url_path(self, base_url, path): - if base_url: - if self.vcs_subdir: - extra_path = self.vcs_subdir + '/' + path - else: - extra_path = path - if '%path%' in base_url: - if extra_path: - url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', base_url) - return url.replace('%path%', extra_path) - else: - url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', base_url) - return url - else: - return base_url + extra_path + def get_layerbranch(self, branchname): + res = list(self.layerbranch_set.filter(branch__name=branchname)[:1]) + if res: + return res[0] return None - def tree_url(self, path = ''): - return self._handle_url_path(self.vcs_web_tree_base_url, path) - - def file_url(self, path = ''): - return self._handle_url_path(self.vcs_web_file_base_url, path) - - def test_tree_url(self): - return self.tree_url('conf') - - def test_file_url(self): - return self.file_url('conf/layer.conf') - - def sorted_recipes(self): - return self.recipe_set.order_by('filename') - def active_maintainers(self): - return self.layermaintainer_set.filter(status='A') + matches = None + for layerbranch in self.layerbranch_set.all(): + branchmatches = layerbranch.layermaintainer_set.filter(status='A') + if matches: + matches |= branchmatches + else: + matches = branchmatches + return matches def user_can_edit(self, user): if user.is_authenticated(): @@ -96,12 +86,62 @@ class LayerItem(models.Model): return self.name +class LayerBranch(models.Model): + layer = models.ForeignKey(LayerItem) + branch = models.ForeignKey(Branch) + vcs_last_fetch = models.DateTimeField('Last successful fetch', blank=True, null=True) + vcs_last_rev = models.CharField('Last revision fetched', max_length=80, blank=True) + vcs_last_commit = models.DateTimeField('Last commit date', blank=True, null=True) + + class Meta: + verbose_name_plural = "Layer branches" + + def sorted_recipes(self): + return self.recipe_set.order_by('filename') + + def active_maintainers(self): + return self.layermaintainer_set.filter(status='A') + + def _handle_url_path(self, base_url, path): + if base_url: + if self.layer.vcs_subdir: + extra_path = self.layer.vcs_subdir + '/' + path + else: + extra_path = path + url = base_url.replace('%branch%', self.branch.name) + if '%path%' in base_url: + if extra_path: + url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '\\1', url) + return url.replace('%path%', extra_path) + else: + url = re.sub(r'\[([^\]]*%path%[^\]]*)\]', '', url) + return url + else: + return url + extra_path + return None + + def tree_url(self, path = ''): + return self._handle_url_path(self.layer.vcs_web_tree_base_url, path) + + def file_url(self, path = ''): + return self._handle_url_path(self.layer.vcs_web_file_base_url, path) + + def test_tree_url(self): + return self.tree_url('conf') + + def test_file_url(self): + return self.file_url('conf/layer.conf') + + def __unicode__(self): + return "%s: %s" % (self.layer.name, self.branch.name) + + class LayerMaintainer(models.Model): MAINTAINER_STATUS_CHOICES = ( ('A', 'Active'), ('I', 'Inactive'), ) - layer = models.ForeignKey(LayerItem) + layerbranch = models.ForeignKey(LayerBranch) name = models.CharField(max_length=255) email = models.CharField(max_length=255) responsibility = models.CharField(max_length=200, blank=True, help_text='Specific area(s) this maintainer is responsible for, if not the entire layer') @@ -111,18 +151,18 @@ class LayerMaintainer(models.Model): respstr = "" if self.responsibility: respstr = " (%s)" % self.responsibility - return "%s: %s <%s>%s" % (self.layer.name, self.name, self.email, respstr) + return "%s: %s <%s>%s" % (self.layerbranch.layer.name, self.name, self.email, respstr) class LayerDependency(models.Model): - layer = models.ForeignKey(LayerItem, related_name='dependencies_set') + layerbranch = models.ForeignKey(LayerBranch, related_name='dependencies_set') dependency = models.ForeignKey(LayerItem, related_name='dependents_set') class Meta: verbose_name_plural = "Layer dependencies" def __unicode__(self): - return "%s depends on %s" % (self.layer.name, self.dependency.name) + return "%s depends on %s" % (self.layerbranch.layer.name, self.dependency.name) class LayerNote(models.Model): @@ -134,7 +174,7 @@ class LayerNote(models.Model): class Recipe(models.Model): - layer = models.ForeignKey(LayerItem) + layerbranch = models.ForeignKey(LayerBranch) filename = models.CharField(max_length=255) filepath = models.CharField(max_length=255, blank=True) pn = models.CharField(max_length=40, blank=True) @@ -146,7 +186,7 @@ class Recipe(models.Model): homepage = models.URLField(blank=True) def vcs_web_url(self): - url = self.layer.file_url(os.path.join(self.filepath, self.filename)) + url = self.layerbranch.file_url(os.path.join(self.filepath, self.filename)) return url or '' def full_path(self): @@ -170,7 +210,7 @@ class Recipe(models.Model): class RecipeFileDependency(models.Model): recipe = models.ForeignKey(Recipe) - layer = models.ForeignKey(LayerItem, related_name='+') + layerbranch = models.ForeignKey(LayerBranch, related_name='+') path = models.CharField(max_length=255, db_index=True) class Meta: @@ -181,13 +221,13 @@ class RecipeFileDependency(models.Model): class Machine(models.Model): - layer = models.ForeignKey(LayerItem) + layerbranch = models.ForeignKey(LayerBranch) name = models.CharField(max_length=255) description = models.CharField(max_length=255) def vcs_web_url(self): - url = self.layer.file_url(os.path.join('conf/machine/%s.conf' % self.name)) + url = self.layerbranch.file_url(os.path.join('conf/machine/%s.conf' % self.name)) return url or '' def __unicode__(self): - return '%s (%s)' % (self.name, self.layer.name) + return '%s (%s)' % (self.name, self.layerbranch.layer.name) diff --git a/layerindex/rawrecipes.txt b/layerindex/rawrecipes.txt index cbb7d67..c1911c9 100644 --- a/layerindex/rawrecipes.txt +++ b/layerindex/rawrecipes.txt @@ -1,2 +1,2 @@ -{% for recipe in recipe_list %}{{ recipe.layer.name }} {{ recipe.pn }} {{ recipe.pv }} {{ recipe.full_path }} +{% for recipe in recipe_list %}{{ recipe.layerbranch.layer.name }} {{ recipe.pn }} {{ recipe.pv }} {{ recipe.full_path }} {% endfor %} diff --git a/layerindex/recipedetail.html b/layerindex/recipedetail.html index f336ba9..b0d366a 100644 --- a/layerindex/recipedetail.html +++ b/layerindex/recipedetail.html @@ -32,6 +32,12 @@
  • Recipe
    • + {% if recipe.layerbranch.branch.name != current_branch %} +
    + This recipe record is for a different branch than the one you have selected. To search for the equivalent recipe on the {{ current_branch }} branch, click here. +
    + {% endif %} +
    {{ machine.name }} {{ machine.description }}
    {{ recipe.name }} {{ recipe.pv }}
    {{ layeritem.name }}{{ layeritem.summary }}{{ layeritem.get_layer_type_display }}
    {{ layerbranch.layer.name }}{{ layerbranch.layer.summary }}{{ layerbranch.layer.get_layer_type_display }} - {{ layeritem.vcs_url }} - {% if layeritem.vcs_web_url %} - + {{ layerbranch.layer.vcs_url }} + {% if layerbranch.layer.vcs_web_url %} + web repo {% endif %} - {% if layeritem.tree_url %} - + {% if layerbranch.tree_url %} + tree diff --git a/layerindex/machines.html b/layerindex/machines.html index cfbca56..7cade25 100644 --- a/layerindex/machines.html +++ b/layerindex/machines.html @@ -52,7 +52,7 @@
    {{ machine.name }} {{ machine.description }}{{ machine.layer.name }}{{ machine.layerbranch.layer.name }}
    @@ -74,7 +80,7 @@ - +
    Layer{{ recipe.layer.name }}{{ recipe.layerbranch.layer.name }} ({{ recipe.layerbranch.branch.name}} branch)
    diff --git a/layerindex/recipes.html b/layerindex/recipes.html index f84f604..ccc3b54 100644 --- a/layerindex/recipes.html +++ b/layerindex/recipes.html @@ -54,7 +54,7 @@ {{ recipe.name }} {{ recipe.pv }} {{ recipe.short_desc }} - {{ recipe.layer.name }} + {{ recipe.layerbranch.layer.name }} {% endfor %} diff --git a/layerindex/submitlayer.html b/layerindex/submitlayer.html index 41b5aa1..274f102 100644 --- a/layerindex/submitlayer.html +++ b/layerindex/submitlayer.html @@ -21,6 +21,10 @@ {% endblock %} +{% block branch_selector %} +
  • Branch: master
    • +{% endblock %} + {% block formtop %}

    Please enter the details for the layer you wish to add to the index.

    diff --git a/layerindex/update.py b/layerindex/update.py index 497445a..b7d1ec9 100755 --- a/layerindex/update.py +++ b/layerindex/update.py @@ -108,7 +108,7 @@ def update_recipe_file(data, path, recipe, layerdir_start, repodir): RecipeFileDependency.objects.filter(recipe=recipe).delete() for filedep in filedeps: recipedep = RecipeFileDependency() - recipedep.layer = recipe.layer + recipedep.layerbranch = recipe.layerbranch recipedep.recipe = recipe recipedep.path = filedep recipedep.save() @@ -132,30 +132,19 @@ def parse_layer_conf(layerdir, data): data = bb.cooker._parse(os.path.join(layerdir, "conf", "layer.conf"), data) data.expandVarref('LAYERDIR') -def setup_bitbake_path(basepath): - # Set path to bitbake lib dir - bitbakedir_env = os.environ.get('BITBAKEDIR', '') - if bitbakedir_env and os.path.exists(bitbakedir_env + '/lib/bb'): - bitbakepath = bitbakedir_env - elif basepath and os.path.exists(basepath + '/bitbake/lib/bb'): - bitbakepath = basepath + '/bitbake' - elif basepath and os.path.exists(basepath + '/../bitbake/lib/bb'): - bitbakepath = os.path.abspath(basepath + '/../bitbake') - else: - # look for bitbake/bin dir in PATH - bitbakepath = None - for pth in os.environ['PATH'].split(':'): - if os.path.exists(os.path.join(pth, '../lib/bb')): - bitbakepath = os.path.abspath(os.path.join(pth, '..')) - break - if not bitbakepath: - if basepath: - logger.error("Unable to find bitbake by searching BITBAKEDIR, specified path '%s' or its parent, or PATH" % basepath) - else: - logger.error("Unable to find bitbake by searching BITBAKEDIR or PATH") - sys.exit(1) - return bitbakepath +def get_branch(branchname): + from layerindex.models import Branch + res = list(Branch.objects.filter(name=branchname)[:1]) + if res: + return res[0] + return None +def get_layer(layername): + from layerindex.models import LayerItem + res = list(LayerItem.objects.filter(name=layername)[:1]) + if res: + return res[0] + return None def main(): if LooseVersion(git.__version__) < '0.3.1': @@ -167,6 +156,9 @@ def main(): usage = """ %prog [options]""") + parser.add_option("-b", "--branch", + help = "Specify branch to update", + action="store", dest="branch", default='master') parser.add_option("-l", "--layer", help = "Specify layers to update (use commas to separate multiple). Default is all published layers.", action="store", dest="layers") @@ -196,39 +188,16 @@ def main(): from django.core.management import setup_environ from django.conf import settings - from layerindex.models import LayerItem, Recipe, RecipeFileDependency, Machine + from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine from django.db import transaction import settings setup_environ(settings) - if len(sys.argv) > 1: - basepath = os.path.abspath(sys.argv[1]) - else: - basepath = None - bitbakepath = setup_bitbake_path(None) - - # Skip sanity checks - os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS' - os.environ['DISABLE_SANITY_CHECKS'] = '1' - - sys.path.extend([bitbakepath + '/lib']) - import bb.tinfoil - import bb.cooker - tinfoil = bb.tinfoil.Tinfoil() - tinfoil.prepare(config_only = True) - - logger.setLevel(options.loglevel) - - # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set - tinfoil.config_data.setVar('SUMMARY', '') - # Clear the default value of DESCRIPTION so that we can see where it's not set - tinfoil.config_data.setVar('DESCRIPTION', '') - # Clear the default value of HOMEPAGE ('unknown') - tinfoil.config_data.setVar('HOMEPAGE', '') - # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - - # why won't they just fix that?!) - tinfoil.config_data.setVar('LICENSE', '') + branch = get_branch(options.branch) + if not branch: + logger.error("Specified branch %s is not valid" % options.branch) + sys.exit(1) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: @@ -251,6 +220,8 @@ def main(): fetchedrepos = [] failedrepos = [] + bitbakepath = os.path.join(fetchdir, 'bitbake') + if not options.nofetch: # Fetch latest metadata from repositories for layer in layerquery: @@ -264,13 +235,57 @@ def main(): if not os.path.exists(repodir): out = runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir) else: - out = runcmd("git pull", repodir) + out = runcmd("git fetch", repodir) except Exception as e: logger.error("fetch failed: %s" % str(e)) failedrepos.append(layer.vcs_url) continue fetchedrepos.append(layer.vcs_url) + logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL) + if not os.path.exists(bitbakepath): + out = runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir) + else: + out = runcmd("git fetch", bitbakepath) + + # Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc) + out = runcmd("git checkout origin/%s" % branch.bitbake_branch, bitbakepath) + out = runcmd("git clean -f -x", bitbakepath) + + # Skip sanity checks + os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS' + os.environ['DISABLE_SANITY_CHECKS'] = '1' + + # Ensure we have OE-Core set up to get some base configuration + core_layer = get_layer(settings.CORE_LAYER_NAME) + if not core_layer: + logger.error("Unable to find core layer %s in database; check CORE_LAYER_NAME setting" % settings.CORE_LAYER_NAME) + sys.exit(1) + core_urldir = sanitise_path(core_layer.vcs_url) + core_repodir = os.path.join(fetchdir, core_urldir) + core_layerdir = os.path.join(core_repodir, core_layer.vcs_subdir) + out = runcmd("git checkout origin/%s" % options.branch, core_repodir) + out = runcmd("git clean -f -x", core_repodir) + os.environ['BBPATH'] = str("%s:%s" % (os.path.realpath('.'), core_layerdir)) + + sys.path.extend([bitbakepath + '/lib']) + import bb.tinfoil + import bb.cooker + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(config_only = True) + + logger.setLevel(options.loglevel) + + # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set + tinfoil.config_data.setVar('SUMMARY', '') + # Clear the default value of DESCRIPTION so that we can see where it's not set + tinfoil.config_data.setVar('DESCRIPTION', '') + # Clear the default value of HOMEPAGE ('unknown') + tinfoil.config_data.setVar('HOMEPAGE', '') + # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - + # why won't they just fix that?!) + tinfoil.config_data.setVar('LICENSE', '') + # Process and extract data from each layer for layer in layerquery: transaction.enter_transaction_management() @@ -282,23 +297,51 @@ def main(): logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url)) transaction.rollback() continue + # Collect repo info repo = git.Repo(repodir) assert repo.bare == False - topcommit = repo.commit('master') + try: + topcommit = repo.commit('origin/%s' % options.branch) + except: + logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, options.branch)) + transaction.rollback() + continue + if layer.vcs_subdir: # Find latest commit in subdirectory # A bit odd to do it this way but apparently there's no other way in the GitPython API - for commit in repo.iter_commits(paths=layer.vcs_subdir): + for commit in repo.iter_commits('origin/%s' % options.branch, paths=layer.vcs_subdir): topcommit = commit break + layerbranch = layer.get_layerbranch(options.branch) + if not layerbranch: + # LayerBranch doesn't exist for this branch, create it + layerbranch = LayerBranch() + layerbranch.layer = layer + layerbranch.branch = branch + layerbranch.save() + layerdir = os.path.join(repodir, layer.vcs_subdir) layerdir_start = os.path.normpath(layerdir) + os.sep - layerrecipes = Recipe.objects.filter(layer=layer) - layermachines = Machine.objects.filter(layer=layer) - if layer.vcs_last_rev != topcommit.hexsha or options.reload: - logger.info("Collecting data for layer %s" % layer.name) + layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) + layermachines = Machine.objects.filter(layerbranch=layerbranch) + if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload: + # Check out appropriate branch + out = runcmd("git checkout origin/%s" % options.branch, repodir) + + if not os.path.exists(layerdir): + if options.branch == 'master': + logger.error("Subdirectory for layer %s does not exist on master branch!" % layer.name) + transaction.rollback() + continue + else: + logger.info("Skipping update of layer %s for branch %s - subdirectory does not exist on this branch" % (layer.name, options.branch)) + transaction.rollback() + continue + + logger.info("Collecting data for layer %s on branch %s" % (layer.name, options.branch)) # Parse layer.conf files for this layer and its dependencies # This is necessary not just because BBPATH needs to be set in order @@ -308,16 +351,16 @@ def main(): config_data_copy = bb.data.createCopy(tinfoil.config_data) parse_layer_conf(layerdir, config_data_copy) - for dep in layer.dependencies_set.all(): + for dep in layerbranch.dependencies_set.all(): depurldir = sanitise_path(dep.dependency.vcs_url) deprepodir = os.path.join(fetchdir, depurldir) deplayerdir = os.path.join(deprepodir, dep.dependency.vcs_subdir) parse_layer_conf(deplayerdir, config_data_copy) config_data_copy.delVar('LAYERDIR') - if layer.vcs_last_rev and not options.reload: + if layerbranch.vcs_last_rev and not options.reload: try: - diff = repo.commit(layer.vcs_last_rev).diff(topcommit) + diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit) except Exception as e: logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e))) diff = None @@ -350,7 +393,7 @@ def main(): (filepath, filename) = split_bb_file_path(path, subdir_start) if filename: recipe = Recipe() - recipe.layer = layer + recipe.layerbranch = layerbranch recipe.filename = filename recipe.filepath = filepath update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir) @@ -360,7 +403,7 @@ def main(): machinename = check_machine_conf(path) if machinename: machine = Machine() - machine.layer = layer + machine.layerbranch = layerbranch machine.name = machinename update_machine_conf_file(os.path.join(repodir, path), machine) machine.save() @@ -385,7 +428,7 @@ def main(): machine = results[0] update_machine_conf_file(os.path.join(repodir, path), machine) machine.save() - deps = RecipeFileDependency.objects.filter(layer=layer).filter(path=path) + deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path) for dep in deps: dirtyrecipes.add(dep.recipe) @@ -400,7 +443,7 @@ def main(): for f in files: if fnmatch.fnmatch(f, "*.bb"): recipe = Recipe() - recipe.layer = layer + recipe.layerbranch = layerbranch recipe.filename = f recipe.filepath = os.path.relpath(root, layerdir) update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir) @@ -410,19 +453,19 @@ def main(): machinename = check_machine_conf(fullpath) if machinename: machine = Machine() - machine.layer = layer + machine.layerbranch = layerbranch machine.name = machinename update_machine_conf_file(fullpath, machine) machine.save() # Save repo info - layer.vcs_last_rev = topcommit.hexsha - layer.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date) + layerbranch.vcs_last_rev = topcommit.hexsha + layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date) else: - logger.info("Layer %s is already up-to-date" % layer.name) + logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, options.branch)) - layer.vcs_last_fetch = datetime.now() - layer.save() + layerbranch.vcs_last_fetch = datetime.now() + layerbranch.save() if options.dryrun: transaction.rollback() diff --git a/layerindex/urls.py b/layerindex/urls.py index b016009..0bc4bee 100644 --- a/layerindex/urls.py +++ b/layerindex/urls.py @@ -7,7 +7,7 @@ from django.conf.urls.defaults import * from django.views.generic import DetailView, ListView from layerindex.models import LayerItem, Recipe -from layerindex.views import LayerListView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view +from layerindex.views import LayerListView, LayerReviewListView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, switch_branch_view urlpatterns = patterns('', url(r'^$', @@ -26,9 +26,7 @@ urlpatterns = patterns('', template_name='layerindex/machines.html'), name='machine_search'), url(r'^review/$', - ListView.as_view( - queryset=LayerItem.objects.order_by('name').filter(status__in='N'), - context_object_name='layer_list', + LayerReviewListView.as_view( template_name='layerindex/index.html'), name='layer_list_review'), url(r'^layer/(?P[-\w]+)/$', @@ -49,6 +47,8 @@ urlpatterns = patterns('', template_name='layerindex/recipedetail.html'), name='recipe'), url(r'^layer/(?P[-\w]+)/publish/$', 'layerindex.views.publish', name="publish"), + url(r'^branch/(?P[-\w]+)/$', + switch_branch_view, name="switch_branch"), url(r'^raw/recipes.txt$', PlainTextListView.as_view( queryset=Recipe.objects.order_by('pn', 'layer'), diff --git a/layerindex/views.py b/layerindex/views.py index e058d9f..67fa4be 100644 --- a/layerindex/views.py +++ b/layerindex/views.py @@ -9,7 +9,7 @@ from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidde from django.core.urlresolvers import reverse from django.core.exceptions import PermissionDenied from django.template import RequestContext -from layerindex.models import LayerItem, LayerMaintainer, LayerDependency, LayerNote, Recipe, Machine +from layerindex.models import Branch, LayerItem, LayerMaintainer, LayerBranch, LayerDependency, LayerNote, Recipe, Machine from datetime import datetime from django.views.generic import DetailView, ListView from layerindex.forms import EditLayerForm, LayerMaintainerFormSet, EditNoteForm @@ -79,39 +79,45 @@ def delete_layer_view(request, template_name, slug): def edit_layer_view(request, template_name, slug=None): if slug: # Edit mode + branch = Branch.objects.filter(name=request.session.get('branch', 'master'))[:1].get() layeritem = get_object_or_404(LayerItem, name=slug) if not (request.user.is_authenticated() and (request.user.has_perm('layerindex.publish_layer') or layeritem.user_can_edit(request.user))): raise PermissionDenied + layerbranch = get_object_or_404(LayerBranch, layer=layeritem, branch=branch) deplistlayers = LayerItem.objects.exclude(id=layeritem.id).order_by('name') else: # Submit mode + branch = Branch.objects.filter(name='master')[:1].get() layeritem = LayerItem() + layerbranch = LayerBranch(layer=layeritem, branch=branch) deplistlayers = LayerItem.objects.all().order_by('name') if request.method == 'POST': - form = EditLayerForm(request.user, request.POST, instance=layeritem) - maintainerformset = LayerMaintainerFormSet(request.POST, instance=layeritem) + form = EditLayerForm(request.user, layerbranch, request.POST, instance=layeritem) + maintainerformset = LayerMaintainerFormSet(request.POST, instance=layerbranch) if form.is_valid() and maintainerformset.is_valid(): with transaction.commit_on_success(): form.save() + layerbranch.layer = layeritem + layerbranch.save() maintainerformset.save() if slug: new_deps = form.cleaned_data['deps'] - existing_deps = [deprec.dependency for deprec in layeritem.dependencies_set.all()] + existing_deps = [deprec.dependency for deprec in layerbranch.dependencies_set.all()] for dep in new_deps: if dep not in existing_deps: deprec = LayerDependency() - deprec.layer = layeritem + deprec.layerbranch = layerbranch deprec.dependency = dep deprec.save() for dep in existing_deps: if dep not in new_deps: - layeritem.dependencies_set.filter(dependency=dep).delete() + layerbranch.dependencies_set.filter(dependency=dep).delete() else: # Save dependencies for dep in form.cleaned_data['deps']: deprec = LayerDependency() - deprec.layer = layeritem + deprec.layerbranch = layerbranch deprec.dependency = dep deprec.save() # Send email @@ -133,8 +139,8 @@ def edit_layer_view(request, template_name, slug=None): return HttpResponseRedirect(reverse('submit_layer_thanks')) form.was_saved = True else: - form = EditLayerForm(request.user, instance=layeritem) - maintainerformset = LayerMaintainerFormSet(instance=layeritem) + form = EditLayerForm(request.user, layerbranch, instance=layeritem) + maintainerformset = LayerMaintainerFormSet(instance=layerbranch) return render(request, template_name, { 'form': form, @@ -145,6 +151,14 @@ def edit_layer_view(request, template_name, slug=None): def submit_layer_thanks(request): return render(request, 'layerindex/submitthanks.html') +def switch_branch_view(request, slug): + branch = get_object_or_404(Branch, name=slug) + request.session['branch'] = branch.name + return_url = request.META.get('HTTP_REFERER') + if not return_url: + return_url = reverse('layer_list') + return HttpResponseRedirect(return_url) + def about(request): return render(request, 'layerindex/about.html') @@ -161,16 +175,20 @@ def _statuschange(request, name, newstatus): return HttpResponseRedirect(w.get_absolute_url()) class LayerListView(ListView): - context_object_name = 'layer_list' + context_object_name = 'layerbranch_list' def get_queryset(self): - return LayerItem.objects.filter(status='P').order_by('name') + return LayerBranch.objects.filter(branch__name=self.request.session.get('branch', 'master')).filter(layer__status='P').order_by('layer__name') def get_context_data(self, **kwargs): context = super(LayerListView, self).get_context_data(**kwargs) context['layer_type_choices'] = LayerItem.LAYER_TYPE_CHOICES return context +class LayerReviewListView(ListView): + def get_queryset(self): + return LayerBranch.objects.filter(branch__name=self.request.session.get('branch', 'master')).filter(layer__status='N').order_by('layer__name') + class LayerDetailView(DetailView): model = LayerItem slug_field = 'name' @@ -190,6 +208,7 @@ class LayerDetailView(DetailView): context = super(LayerDetailView, self).get_context_data(**kwargs) layer = context['layeritem'] context['useredit'] = layer.user_can_edit(self.user) + context['layerbranch'] = layer.get_layerbranch(self.request.session.get('branch', 'master')) return context class RecipeSearchView(ListView): @@ -198,11 +217,12 @@ class RecipeSearchView(ListView): def get_queryset(self): query_string = self.request.GET.get('q', '') + init_qs = Recipe.objects.filter(layerbranch__branch__name=self.request.session.get('branch', 'master')) if query_string.strip(): entry_query = simplesearch.get_query(query_string, ['pn', 'summary', 'description', 'filename']) - return Recipe.objects.filter(entry_query).order_by('pn', 'layer') + return init_qs.filter(entry_query).order_by('pn', 'layerbranch__layer') else: - return Recipe.objects.all().order_by('pn', 'layer') + return init_qs.order_by('pn', 'layerbranch__layer') def get_context_data(self, **kwargs): context = super(RecipeSearchView, self).get_context_data(**kwargs) @@ -215,11 +235,12 @@ class MachineSearchView(ListView): def get_queryset(self): query_string = self.request.GET.get('q', '') + init_qs = Machine.objects.filter(layerbranch__branch__name=self.request.session.get('branch', 'master')) if query_string.strip(): entry_query = simplesearch.get_query(query_string, ['name', 'description']) - return Machine.objects.filter(entry_query).order_by('name', 'layer') + return init_qs.filter(entry_query).order_by('name', 'layerbranch__layer') else: - return Machine.objects.all().order_by('name', 'layer') + return init_qs.order_by('name', 'layerbranch__layer') def get_context_data(self, **kwargs): context = super(MachineSearchView, self).get_context_data(**kwargs) diff --git a/settings.py b/settings.py index 3dbe232..f9da685 100644 --- a/settings.py +++ b/settings.py @@ -115,6 +115,7 @@ X_FRAME_OPTIONS = 'DENY' from django.conf.global_settings import TEMPLATE_CONTEXT_PROCESSORS as TCP TEMPLATE_CONTEXT_PROCESSORS = TCP + ( 'django.core.context_processors.request', + 'layerindex.context_processors.layerindex_context', ) ROOT_URLCONF = 'urls' @@ -175,6 +176,12 @@ LOGIN_REDIRECT_URL = '/layerindex' # Full path to directory where layers should be fetched into by the update script LAYER_FETCH_DIR = "" +# Fetch URL of the BitBake repository for the update script +BITBAKE_REPO_URL = "git://git.openembedded.org/bitbake" + +# Core layer to be used by the update script for basic BitBake configuration +CORE_LAYER_NAME = "openembedded-core" + # Settings for layer submission feature SUBMIT_EMAIL_FROM = 'noreply@example.com' SUBMIT_EMAIL_SUBJECT = 'OE Layerindex layer submission'