poky/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
Richard Purdie 9501864db8 bitbake: bitbake: Strip old editor directives from file headers
There are much better ways to handle this and most editors shouldn't need this
in modern times, drop the noise from the files. Its not consitently applied
anyway.

(Bitbake rev: 5e43070e3087d09aea2f459b033d035c5ef747d0)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2019-05-04 10:44:10 +01:00

572 lines
24 KiB
Python

#
# BitBake Toaster Implementation
#
# Copyright (C) 2018 Wind River Systems
#
# SPDX-License-Identifier: GPL-2.0-only
#
# buildimport: import a project for project specific configuration
#
# Usage:
# (a) Set up Toaster environent
#
# (b) Call buildimport
# $ /path/to/bitbake/lib/toaster/manage.py buildimport \
# --name=$PROJECTNAME \
# --path=$BUILD_DIRECTORY \
# --callback="$CALLBACK_SCRIPT" \
# --command="configure|reconfigure|import"
#
# (c) Return is "|Default_image=%s|Project_id=%d"
#
# (d) Open Toaster to this project using for example:
# $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
#
# (e) To delete a project:
# $ /path/to/bitbake/lib/toaster/manage.py buildimport \
# --name=$PROJECTNAME --delete-project
#
# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
from django.core.management.base import BaseCommand, CommandError
from django.core.exceptions import ObjectDoesNotExist
from orm.models import ProjectManager, Project, Release, ProjectVariable
from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
from toastergui.api import scan_layer_content
from django.db import OperationalError
import os
import re
import os.path
import subprocess
import shutil
# Toaster variable section delimiters
TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
# quick development/debugging support
verbose = 2
def _log(msg):
if 1 == verbose:
print(msg)
elif 2 == verbose:
f1=open('/tmp/toaster.log', 'a')
f1.write("|" + msg + "|\n" )
f1.close()
__config_regexp__ = re.compile( r"""
^
(?P<exp>export\s+)?
(?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
(\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
\s* (
(?P<colon>:=) |
(?P<lazyques>\?\?=) |
(?P<ques>\?=) |
(?P<append>\+=) |
(?P<prepend>=\+) |
(?P<predot>=\.) |
(?P<postdot>\.=) |
=
) \s*
(?!'[^']*'[^']*'$)
(?!\"[^\"]*\"[^\"]*\"$)
(?P<apo>['\"])
(?P<value>.*)
(?P=apo)
$
""", re.X)
class Command(BaseCommand):
args = "<name> <path> <release>"
help = "Import a command line build directory"
vars = {}
toaster_vars = {}
def add_arguments(self, parser):
parser.add_argument(
'--name', dest='name', required=True,
help='name of the project',
)
parser.add_argument(
'--path', dest='path', required=True,
help='path to the project',
)
parser.add_argument(
'--release', dest='release', required=False,
help='release for the project',
)
parser.add_argument(
'--callback', dest='callback', required=False,
help='callback for project config update',
)
parser.add_argument(
'--delete-project', dest='delete_project', required=False,
help='delete this project from the database',
)
parser.add_argument(
'--command', dest='command', required=False,
help='command (configure,reconfigure,import)',
)
# Extract the bb variables from a conf file
def scan_conf(self,fn):
vars = self.vars
toaster_vars = self.toaster_vars
#_log("scan_conf:%s" % fn)
if not os.path.isfile(fn):
return
f = open(fn, 'r')
#statements = ast.StatementGroup()
lineno = 0
is_toaster_section = False
while True:
lineno = lineno + 1
s = f.readline()
if not s:
break
w = s.strip()
# skip empty lines
if not w:
continue
# evaluate Toaster sections
if w.startswith(TOASTER_PROLOG):
is_toaster_section = True
continue
if w.startswith(TOASTER_EPILOG):
is_toaster_section = False
continue
s = s.rstrip()
while s[-1] == '\\':
s2 = f.readline().strip()
lineno = lineno + 1
if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
s = s[:-1] + s2
# skip comments
if s[0] == '#':
continue
# process the line for just assignments
m = __config_regexp__.match(s)
if m:
groupd = m.groupdict()
var = groupd['var']
value = groupd['value']
if groupd['lazyques']:
if not var in vars:
vars[var] = value
continue
if groupd['ques']:
if not var in vars:
vars[var] = value
continue
# preset empty blank for remaining operators
if not var in vars:
vars[var] = ''
if groupd['append']:
vars[var] += value
elif groupd['prepend']:
vars[var] = "%s%s" % (value,vars[var])
elif groupd['predot']:
vars[var] = "%s %s" % (value,vars[var])
elif groupd['postdot']:
vars[var] = "%s %s" % (vars[var],value)
else:
vars[var] = "%s" % (value)
# capture vars in a Toaster section
if is_toaster_section:
toaster_vars[var] = vars[var]
# DONE WITH PARSING
f.close()
self.vars = vars
self.toaster_vars = toaster_vars
# Update the scanned project variables
def update_project_vars(self,project,name):
pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
if (not name in self.vars.keys()) or (not self.vars[name]):
self.vars[name] = pv.value
else:
if pv.value != self.vars[name]:
pv.value = self.vars[name]
pv.save()
# Find the git version of the installation
def find_layer_dir_version(self,path):
# * rocko ...
install_version = ''
cwd = os.getcwd()
os.chdir(path)
p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
out = out.decode("utf-8")
for branch in out.split('\n'):
if ('*' == branch[0:1]) and ('no branch' not in branch):
install_version = re.sub(' .*','',branch[2:])
break
if 'remotes/m/master' in branch:
install_version = re.sub('.*base/','',branch)
break
os.chdir(cwd)
return install_version
# Compute table of the installation's registered layer versions (branch or commit)
def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
lv_dict = {}
layer_versions = Layer_Version.objects.all()
for lv in layer_versions:
layer = Layer.objects.filter(pk=lv.layer.pk)[0]
if layer.vcs_url:
url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
else:
url_short = ''
# register the core, branch, and the version variations
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
#_log(" (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
return lv_dict
# Apply table of all layer versions
def extract_bblayers(self):
# set up the constants
bblayer_str = self.vars['BBLAYERS']
TOASTER_DIR = os.environ.get('TOASTER_DIR')
INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
INSTALL_URL_PREFIX = ''
layers = Layer.objects.filter(name='openembedded-core')
for layer in layers:
if layer.vcs_url:
INSTALL_URL_PREFIX = layer.vcs_url
break
INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
INSTALL_VERSION_DIR = TOASTER_DIR
INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
# get the set of available layer:layer_versions
lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
# compute the layer matches
layers_list = []
for line in bblayer_str.split(' '):
if not line:
continue
if line.endswith('/local'):
continue
# isolate the repo
layer_path = line
line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
# isolate the sub-path
path_index = line.rfind('/')
if path_index > 0:
sub_path = line[path_index+1:]
line = line[0:path_index]
else:
sub_path = ''
# isolate the version
if TOASTER_CLONE_PREFIX in layer_path:
is_toaster_clone = True
# extract version from name syntax
version_index = line.find('_')
if version_index > 0:
version = line[version_index+1:]
line = line[0:version_index]
else:
version = ''
_log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
else:
is_toaster_clone = False
# version is from the installation
version = self.find_layer_dir_version(layer_path)
_log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
# capture the layer information into layers_list
layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
return layers_list,lv_dict
#
def find_import_release(self,layers_list,lv_dict,default_release):
# poky,meta,rocko => 4;openembedded-core
release = default_release
for line,path,version,layer_path,is_toaster_clone in layers_list:
key = "%s,%s,%s" % (line,path,version)
if key in lv_dict:
lv_id = lv_dict[key]
if 'openembedded-core' == lv_id[1]:
_log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
# only versions in Toaster managed layers are accepted
if not is_toaster_clone:
break
try:
release = Release.objects.get(name=version)
except:
pass
break
_log("Find_import_release:RELEASE=%s" % release.name)
return release
# Apply the found conf layers
def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
for line,path,version,layer_path,is_toaster_clone in layers_list:
# Assert release promote if present
if release:
version = release
# try to match the key to a layer_version
key = "%s,%s,%s" % (line,path,version)
key_short = "%s,%s,%s" % (line,path,'')
lv_id = ''
if key in lv_dict:
lv_id = lv_dict[key]
lv = Layer_Version.objects.get(pk=int(lv_id[0]))
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=lv)
pl.optional=False
pl.save()
_log(" %s => %s;%s" % (key,lv_id[0],lv_id[1]))
elif key_short in lv_dict:
lv_id = lv_dict[key_short]
lv = Layer_Version.objects.get(pk=int(lv_id[0]))
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=lv)
pl.optional=False
pl.save()
_log(" %s ?> %s" % (key,lv_dict[key_short]))
else:
_log("%s <= %s" % (key,layer_path))
found = False
# does local layer already exist in this project?
try:
for pl in ProjectLayer.objects.filter(project=project):
if pl.layercommit.layer.local_source_dir == layer_path:
found = True
_log(" Project Local Layer found!")
except Exception as e:
_log("ERROR: Local Layer '%s'" % e)
pass
if not found:
# Does Layer name+path already exist?
try:
layer_name_base = os.path.basename(layer_path)
_log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
# Found! Attach layer_version and ProjectLayer
layer_version = Layer_Version.objects.create(
layer=layer,
project=project,
layer_source=LayerSource.TYPE_IMPORTED)
layer_version.save()
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=layer_version)
pl.optional=False
pl.save()
found = True
# add layer contents to this layer version
scan_layer_content(layer,layer_version)
_log(" Parent Local Layer found in db!")
except Exception as e:
_log("Layer_exists_test_failed: Local Layer '%s'" % e)
pass
if not found:
# Insure that layer path exists, in case of user typo
if not os.path.isdir(layer_path):
_log("ERROR:Layer path '%s' not found" % layer_path)
continue
# Add layer to db and attach project to it
layer_name_base = os.path.basename(layer_path)
# generate a unique layer name
layer_name_matches = {}
for layer in Layer.objects.filter(name__contains=layer_name_base):
layer_name_matches[layer.name] = '1'
layer_name_idx = 0
layer_name_test = layer_name_base
while layer_name_test in layer_name_matches.keys():
layer_name_idx += 1
layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
# create the layer and layer_verion objects
layer = Layer.objects.create(name=layer_name_test)
layer.local_source_dir = layer_path
layer_version = Layer_Version.objects.create(
layer=layer,
project=project,
layer_source=LayerSource.TYPE_IMPORTED)
layer.save()
layer_version.save()
pl,created = ProjectLayer.objects.get_or_create(project=project,
layercommit=layer_version)
pl.optional=False
pl.save()
# register the layer's content
_log(" Local Layer Add content")
scan_layer_content(layer,layer_version)
_log(" Local Layer Added '%s'!" % layer_name_test)
# Scan the project's conf files (if any)
def scan_conf_variables(self,project_path):
# scan the project's settings, add any new layers or variables
if os.path.isfile("%s/conf/local.conf" % project_path):
self.scan_conf("%s/conf/local.conf" % project_path)
self.scan_conf("%s/conf/bblayers.conf" % project_path)
# Import then disable old style Toaster conf files (before 'merged_attr')
old_toaster_local = "%s/conf/toaster.conf" % project_path
if os.path.isfile(old_toaster_local):
self.scan_conf(old_toaster_local)
shutil.move(old_toaster_local, old_toaster_local+"_old")
old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
if os.path.isfile(old_toaster_layer):
self.scan_conf(old_toaster_layer)
shutil.move(old_toaster_layer, old_toaster_layer+"_old")
# Scan the found conf variables (if any)
def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
if self.vars:
# Catch vars relevant to Toaster (in case no Toaster section)
self.update_project_vars(project,'DISTRO')
self.update_project_vars(project,'MACHINE')
self.update_project_vars(project,'IMAGE_INSTALL_append')
self.update_project_vars(project,'IMAGE_FSTYPES')
self.update_project_vars(project,'PACKAGE_CLASSES')
# These vars are typically only assigned by Toaster
#self.update_project_vars(project,'DL_DIR')
#self.update_project_vars(project,'SSTATE_DIR')
# Assert found Toaster vars
for var in self.toaster_vars.keys():
pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
pv.value = self.toaster_vars[var]
_log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
pv.save()
# Assert found BBLAYERS
if 0 < verbose:
for pl in ProjectLayer.objects.filter(project=project):
release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
self.apply_conf_bblayers(layers_list,lv_dict,project,release)
if 0 < verbose:
for pl in ProjectLayer.objects.filter(project=project):
release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
def handle(self, *args, **options):
project_name = options['name']
project_path = options['path']
project_callback = options['callback'] if options['callback'] else ''
release_name = options['release'] if options['release'] else ''
#
# Delete project
#
if options['delete_project']:
try:
print("Project '%s' delete from Toaster database" % (project_name))
project = Project.objects.get(name=project_name)
# TODO: deep project delete
project.delete()
print("Project '%s' Deleted" % (project_name))
return
except Exception as e:
print("Project '%s' not found, not deleted (%s)" % (project_name,e))
return
#
# Create/Update/Import project
#
# See if project (by name) exists
project = None
try:
# Project already exists
project = Project.objects.get(name=project_name)
except Exception as e:
pass
# Find the installation's default release
default_release = Release.objects.get(id=1)
# SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
if ("reconfigure" == options['command']) and (None == project):
options['command'] = 'import'
# 'Configure':
if "configure" == options['command']:
# Note: ignore any existing conf files
# create project, SANITY: reuse any project of same name
project = Project.objects.create_project(project_name,default_release,project)
# 'Re-configure':
if "reconfigure" == options['command']:
# Scan the directory's conf files
self.scan_conf_variables(project_path)
# Scan the layer list
layers_list,lv_dict = self.extract_bblayers()
# Apply any new layers or variables
self.apply_conf_variables(project,layers_list,lv_dict)
# 'Import':
if "import" == options['command']:
# Scan the directory's conf files
self.scan_conf_variables(project_path)
# Remove these Toaster controlled variables
for var in ('DL_DIR','SSTATE_DIR'):
self.vars.pop(var, None)
self.toaster_vars.pop(var, None)
# Scan the layer list
layers_list,lv_dict = self.extract_bblayers()
# Find the directory's release, and promote to default_release if local paths
release = self.find_import_release(layers_list,lv_dict,default_release)
# create project, SANITY: reuse any project of same name
project = Project.objects.create_project(project_name,release,project)
# Apply any new layers or variables
self.apply_conf_variables(project,layers_list,lv_dict,release)
# WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
# Set up the project's meta data
project.builddir = project_path
project.merged_attr = True
project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
if ("configure" == options['command']) or ("import" == options['command']):
# preset the mode and default image recipe
project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
# Assert any extended/custom actions or variables for new non-Toaster projects
if not len(self.toaster_vars):
pass
else:
project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
# Save the updated Project
project.save()
_log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
else:
print("|Project_id=%d" % (project.id))