Support (and require) Python 3

We need to be able to support Python 3 so that we can parse master of
OE-Core with bitbake (which now requires it). This now means the
interface itself and the update script require Python 3.4+.

Part of the implementation for [YOCTO #9704].

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>

README

@@ -11,6 +11,7 @@ Setup
 
 In order to make use of this application you will need:
 
+* Python 3.4+
 * Django 1.6.x - tested with 1.6.10; newer versions may work, but
   the application has not been tested with 1.7 or newer.
 * For production usage, a web server set up to host Django applications
@@ -32,8 +33,9 @@ In order to make use of this application you will need:
   have to have Django installed, have the same or similar configuration
   in settings.py and have access to the database used by the web
   application):
-  * Python 2.7.3
-  * GitPython (python-git) version 0.3.1 or later
+  * Python 2.7.6+ / Python 3.4+ to match with the version of BitBake
+    for the OpenEmbedded branch being parsed
+  * GitPython (python-git) version 2.0 or later
 
 Setup instructions:
 

layerindex/bulkchange.py

@@ -50,7 +50,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
                     outfile = open(os.path.join(tmpoutdir, patchname), 'w')
                     last_layer = layer
                 recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename))
-                varlist = list(set(fields.keys() + meta_vars))
+                varlist = list(set(list(fields.keys()) + meta_vars))
                 varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy)
                 filevars = localise_file_vars(recipefile, varfiles, fields.keys())
                 for f, fvars in filevars.items():
@@ -66,7 +66,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
     ret = None
     if len(patches) > 1:
         (tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir)
-        tmptarfile = os.fdopen(tmptarfd, "w")
+        tmptarfile = os.fdopen(tmptarfd, "wb")
         tar = tarfile.open(None, "w:gz", tmptarfile)
         for patch in patches:
             patchfn = os.path.join(tmpoutdir, patch)
@@ -75,7 +75,7 @@ def generate_patches(tinfoil, fetchdir, changeset, outputdir):
         ret = tmptarname
     elif len(patches) == 1:
         (tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir)
-        tmppatchfile = os.fdopen(tmppatchfd, "w")
+        tmppatchfile = os.fdopen(tmppatchfd, "wb")
         with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile:
             shutil.copyfileobj(patchfile, tmppatchfile)
         tmppatchfile.close()
@@ -93,7 +93,7 @@ def patch_recipe(fn, relpath, values):
     remainingnames = {}
     for k in values.keys():
         remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
-    remainingnames = SortedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))
+    remainingnames = SortedDict(sorted(remainingnames.items(), key=lambda x: x[1]))
 
     with tempfile.NamedTemporaryFile('w', delete=False) as tf:
         def outputvalue(name):
@@ -234,7 +234,7 @@ def main():
         utils.unlock_file(lockfile)
 
     if outp:
-        print outp
+        print(outp)
     else:
         sys.stderr.write("No changes to write\n")
         sys.exit(1)

layerindex/models.py

@@ -26,7 +26,7 @@ class Branch(models.Model):
     class Meta:
         verbose_name_plural = "Branches"
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 
@@ -106,7 +106,7 @@ class LayerItem(models.Model):
     def get_absolute_url(self):
         return reverse('layer_item', args=('master',self.name));
 
-    def __unicode__(self):
+    def __str__(self):
         return self.name
 
 
@@ -188,7 +188,7 @@ class LayerBranch(models.Model):
                     url = resolveComponents(url)
             return url
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s" % (self.layer.name, self.branch.name)
 
 
@@ -203,7 +203,7 @@ class LayerMaintainer(models.Model):
     responsibility = models.CharField(max_length=200, blank=True, help_text='Specific area(s) this maintainer is responsible for, if not the entire layer')
     status = models.CharField(max_length=1, choices=MAINTAINER_STATUS_CHOICES, default='A')
 
-    def __unicode__(self):
+    def __str__(self):
         respstr = ""
         if self.responsibility:
             respstr = " (%s)" % self.responsibility
@@ -217,7 +217,7 @@ class LayerDependency(models.Model):
     class Meta:
         verbose_name_plural = "Layer dependencies"
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s depends on %s" % (self.layerbranch.layer.name, self.dependency.name)
 
 
@@ -225,7 +225,7 @@ class LayerNote(models.Model):
     layer = models.ForeignKey(LayerItem)
     text = models.TextField()
 
-    def __unicode__(self):
+    def __str__(self):
         return "%s: %s" % (self.layer.name, self.text)
 
 
@@ -266,7 +266,7 @@ class Recipe(models.Model):
         else:
             return self.filename.split('_')[0]
 
-    def __unicode__(self):
+    def __str__(self):
         return os.path.join(self.filepath, self.filename)
 
 
@@ -278,7 +278,7 @@ class RecipeFileDependency(models.Model):
     class Meta:
         verbose_name_plural = "Recipe file dependencies"
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s' % self.path
 
 
@@ -341,7 +341,7 @@ class Machine(models.Model):
         url = self.layerbranch.file_url(os.path.join('conf/machine/%s.conf' % self.name))
         return url or ''
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s (%s)' % (self.name, self.layerbranch.layer.name)
 
 
@@ -367,7 +367,7 @@ class BBAppend(models.Model):
             return fnmatch.fnmatch(recipename, appendname.replace('%', '*'))
         return False
 
-    def __unicode__(self):
+    def __str__(self):
         return os.path.join(self.filepath, self.filename)
 
 
@@ -383,7 +383,7 @@ class BBClass(models.Model):
         url = self.layerbranch.file_url(os.path.join('classes', "%s.bbclass" % self.name))
         return url or ''
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s (%s)' % (self.name, self.layerbranch.layer.name)
 
 
@@ -391,7 +391,7 @@ class RecipeChangeset(models.Model):
     user = models.ForeignKey(User)
     name = models.CharField(max_length=255)
 
-    def __unicode__(self):
+    def __str__(self):
         return '%s' % (self.name)
 
 

layerindex/recipedesc.py

@@ -82,10 +82,10 @@ def main():
         layerrecipes = Recipe.objects.filter(layer=layer)
         for recipe in layerrecipes:
             fullpath = str(os.path.join(repodir, layer.vcs_subdir, recipe.filepath, recipe.filename))
-            print fullpath
+            print(fullpath)
             try:
                 envdata = bb.cache.Cache.loadDataFull(fullpath, [], tinfoil.config_data)
-                print "DESCRIPTION = \"%s\"" % envdata.getVar("DESCRIPTION", True)
+                print("DESCRIPTION = \"%s\"" % envdata.getVar("DESCRIPTION", True))
             except Exception as e:
                 logger.info("Unable to read %s: %s", fullpath, str(e))
 

layerindex/templatetags/addurlparameter.py

@@ -23,7 +23,7 @@ def addurlparameter(parser, token):
     from re import split
     bits = split(r'\s+', token.contents, 2)
     if len(bits) < 2:
-        raise TemplateSyntaxError, "'%s' tag requires two arguments" % bits[0]
+        raise TemplateSyntaxError("'%s' tag requires two arguments" % bits[0])
     return AddParameter(bits[1],bits[2])
 
 register.tag('addurlparameter', addurlparameter)

layerindex/tools/import_layer.py

@@ -164,7 +164,7 @@ def get_github_layerinfo(layer_url, username = None, password = None):
         data = resp.read()
         json_data = json.loads(data)
         #headers = dict((key, value) for key, value in resp.getheaders())
-        #print headers
+        #print(headers)
         owner_resp = github_api_call(json_data['owner']['url'].split('api.github.com')[-1])
         if resp.status in [200, 302]:
             owner_data = owner_resp.read()

layerindex/update.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Fetch layer repositories and update layer index database
 #

layerindex/urls.py

@@ -11,7 +11,7 @@ from django.core.urlresolvers import reverse_lazy
 from layerindex.views import LayerListView, LayerReviewListView, LayerReviewDetailView, RecipeSearchView, MachineSearchView, PlainTextListView, LayerDetailView, edit_layer_view, delete_layer_view, edit_layernote_view, delete_layernote_view, HistoryListView, EditProfileFormView, AdvancedRecipeSearchView, BulkChangeView, BulkChangeSearchView, bulk_change_edit_view, bulk_change_patch_view, BulkChangeDeleteView, RecipeDetailView, RedirectParamsView, ClassicRecipeSearchView, ClassicRecipeDetailView, ClassicRecipeStatsView
 from layerindex.models import LayerItem, Recipe, RecipeChangeset
 from rest_framework import routers
-import restviews
+from . import restviews
 from django.conf.urls import include
 
 router = routers.DefaultRouter()

layerindex/utils.py

@@ -6,7 +6,8 @@
 # Licensed under the MIT license, see COPYING.MIT for details
 
 import sys
-import os.path
+import os
+import tempfile
 import subprocess
 import logging
 import time
@@ -32,10 +33,10 @@ def runcmd(cmd, destdir=None, printerr=True, logger=None):
         return output if succeed
     """
     #logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
-    out = os.tmpfile()
+    out = tempfile.TemporaryFile()
     try:
         subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
-    except subprocess.CalledProcessError,e:
+    except subprocess.CalledProcessError as e:
         out.seek(0)
         if printerr:
             output = out.read()

layerindex/views.py

@@ -4,6 +4,7 @@
 #
 # Licensed under the MIT license, see COPYING.MIT for details
 
+import sys
 from django.shortcuts import get_object_or_404, render
 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404
 from django.core.urlresolvers import reverse, reverse_lazy, resolve
@@ -26,7 +27,7 @@ from django.utils.decorators import method_decorator
 from django.contrib.auth.decorators import login_required
 from django.contrib import messages
 from reversion.models import Revision
-import simplesearch
+from . import simplesearch
 import settings
 from django.dispatch import receiver
 import reversion
@@ -212,14 +213,14 @@ def bulk_change_edit_view(request, template_name, pk):
 def bulk_change_patch_view(request, pk):
     import os
     import os.path
-    import utils
+    from layerindex.utils import runcmd
     changeset = get_object_or_404(RecipeChangeset, pk=pk)
     # FIXME this couples the web server and machine running the update script together,
     # but given that it's a separate script the way is open to decouple them in future
     try:
-        ret = utils.runcmd('python bulkchange.py %d %s' % (int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
+        ret = runcmd('%s bulkchange.py %d %s' % (sys.executable, int(pk), settings.TEMP_BASE_DIR), os.path.dirname(__file__))
         if ret:
-            fn = ret.splitlines()[-1]
+            fn = ret.splitlines()[-1].decode('utf-8')
             if os.path.exists(fn):
                 if fn.endswith('.tar.gz'):
                     mimetype = 'application/x-gzip'