bitbake: toaster: Added new feature to import eventlogs from command line into toaster using replay functionality

Added a new button on the base template to access a new template.
Added a model register the information on the builds and generate access links
Added a form to include the option to load specific files
Added jquery and ajax functions to block screen and redirect to build page when import eventlogs is trigger
Added a new button on landing page linked to import build page, and set min-height of buttons in landing page for uniformity
Removed test assertion to check command line build in content, because new button contains text
Updated toaster_eventreplay to use library
Fix test in test_layerdetails_page
Rebased from master

This feature uses the value from the variable BB_DEFAULT_EVENTLOG to read the files created by bitbake
Exclude listing of files that don't contain the allvariables definitions used to replay builds
This part of the feature should be revisited. Over a long period of time, the BB_DEFAULT_EVENTLOG
will exponentially increase the size of the log file and cause bottlenecks when importing.

(Bitbake rev: ab96cafe03d8bab33c1de09602cc62bd6974f157)

Signed-off-by: Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Marlon Rodriguez Garcia 2023-12-11 11:47:05 -05:00 committed by Richard Purdie
parent 4bb222e0d7
commit df5c8d6471
14 changed files with 543 additions and 81 deletions

View File

@ -30,77 +30,7 @@ sys.path.insert(0, join(dirname(dirname(abspath(__file__))), 'lib'))
import bb.cooker
from bb.ui import toasterui
class EventPlayer:
"""Emulate a connection to a bitbake server."""
def __init__(self, eventfile, variables):
self.eventfile = eventfile
self.variables = variables
self.eventmask = []
def waitEvent(self, _timeout):
"""Read event from the file."""
line = self.eventfile.readline().strip()
if not line:
return
try:
decodedline = json.loads(line)
if 'allvariables' in decodedline:
self.variables = decodedline['allvariables']
return
if not 'vars' in decodedline:
raise ValueError
event_str = decodedline['vars'].encode('utf-8')
event = pickle.loads(codecs.decode(event_str, 'base64'))
event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
if event_name not in self.eventmask:
return
return event
except ValueError as err:
print("Failed loading ", line)
raise err
def runCommand(self, command_line):
"""Emulate running a command on the server."""
name = command_line[0]
if name == "getVariable":
var_name = command_line[1]
variable = self.variables.get(var_name)
if variable:
return variable['v'], None
return None, "Missing variable %s" % var_name
elif name == "getAllKeysWithFlags":
dump = {}
flaglist = command_line[1]
for key, val in self.variables.items():
try:
if not key.startswith("__"):
dump[key] = {
'v': val['v'],
'history' : val['history'],
}
for flag in flaglist:
dump[key][flag] = val[flag]
except Exception as err:
print(err)
return (dump, None)
elif name == 'setEventMask':
self.eventmask = command_line[-1]
return True, None
else:
raise Exception("Command %s not implemented" % command_line[0])
def getEventHandle(self):
"""
This method is called by toasterui.
The return value is passed to self.runCommand but not used there.
"""
pass
from bb.ui import eventreplay
def main(argv):
with open(argv[-1]) as eventfile:
@ -116,7 +46,7 @@ def main(argv):
sys.exit("Cannot find allvariables entry in event log file %s" % argv[-1])
eventfile.seek(0)
params = namedtuple('ConfigParams', ['observe_only'])(True)
player = EventPlayer(eventfile, variables)
player = eventreplay.EventPlayer(eventfile, variables)
return toasterui.main(player, player, params)

View File

@ -0,0 +1,86 @@
#!/usr/bin/env python3
#
# SPDX-License-Identifier: GPL-2.0-only
#
# This file re-uses code spread throughout other Bitbake source files.
# As such, all other copyrights belong to their own right holders.
#
import os
import sys
import json
import pickle
import codecs
class EventPlayer:
"""Emulate a connection to a bitbake server."""
def __init__(self, eventfile, variables):
self.eventfile = eventfile
self.variables = variables
self.eventmask = []
def waitEvent(self, _timeout):
"""Read event from the file."""
line = self.eventfile.readline().strip()
if not line:
return
try:
decodedline = json.loads(line)
if 'allvariables' in decodedline:
self.variables = decodedline['allvariables']
return
if not 'vars' in decodedline:
raise ValueError
event_str = decodedline['vars'].encode('utf-8')
event = pickle.loads(codecs.decode(event_str, 'base64'))
event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
if event_name not in self.eventmask:
return
return event
except ValueError as err:
print("Failed loading ", line)
raise err
def runCommand(self, command_line):
"""Emulate running a command on the server."""
name = command_line[0]
if name == "getVariable":
var_name = command_line[1]
variable = self.variables.get(var_name)
if variable:
return variable['v'], None
return None, "Missing variable %s" % var_name
elif name == "getAllKeysWithFlags":
dump = {}
flaglist = command_line[1]
for key, val in self.variables.items():
try:
if not key.startswith("__"):
dump[key] = {
'v': val['v'],
'history' : val['history'],
}
for flag in flaglist:
dump[key][flag] = val[flag]
except Exception as err:
print(err)
return (dump, None)
elif name == 'setEventMask':
self.eventmask = command_line[-1]
return True, None
else:
raise Exception("Command %s not implemented" % command_line[0])
def getEventHandle(self):
"""
This method is called by toasterui.
The return value is passed to self.runCommand but not used there.
"""
pass

View File

@ -385,7 +385,7 @@ def main(server, eventHandler, params):
main.shutdown = 1
logger.info("ToasterUI build done, brbe: %s", brbe)
continue
break
if isinstance(event, (bb.command.CommandCompleted,
bb.command.CommandFailed,

View File

@ -0,0 +1,22 @@
# Generated by Django 4.2.5 on 2023-11-23 18:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('orm', '0020_models_bigautofield'),
]
operations = [
migrations.CreateModel(
name='EventLogsImports',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=255)),
('imported', models.BooleanField(default=False)),
('build_id', models.IntegerField(blank=True, null=True)),
],
),
]

View File

@ -1868,6 +1868,15 @@ class Distro(models.Model):
def __unicode__(self):
return "Distro " + self.name + "(" + self.description + ")"
class EventLogsImports(models.Model):
name = models.CharField(max_length=255)
imported = models.BooleanField(default=False)
build_id = models.IntegerField(blank=True, null=True)
def __str__(self):
return self.name
django.db.models.signals.post_save.connect(invalidate_cache)
django.db.models.signals.post_delete.connect(invalidate_cache)
django.db.models.signals.m2m_changed.connect(invalidate_cache)

View File

@ -219,5 +219,3 @@ class TestLandingPage(SeleniumTestCase):
content = self.get_page_source()
self.assertTrue(self.PROJECT_NAME in content,
'should show builds for project %s' % self.PROJECT_NAME)
self.assertFalse(self.CLI_BUILDS_PROJECT_NAME in content,
'should not show builds for cli project')

View File

@ -108,7 +108,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
self.wait_until_visible("#save-changes-for-switch", poll=3)
btn_save_chg_for_switch = self.find("#save-changes-for-switch")
btn_save_chg_for_switch.click()
self.driver.execute_script("arguments[0].click();", btn_save_chg_for_switch)
self.wait_until_visible("#edit-layer-source")
# Refresh the page to see if the new values are returned

View File

@ -0,0 +1,14 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# BitBake Toaster UI tests implementation
#
# Copyright (C) 2023 Savoir-faire Linux
#
# SPDX-License-Identifier: GPL-2.0-only
#
from django import forms
from django.core.validators import FileExtensionValidator
class LoadFileForm(forms.Form):
eventlog_file = forms.FileField(widget=forms.FileInput(attrs={'accept': '.json'}))

View File

@ -367,3 +367,31 @@ h2.panel-title { font-size: 30px; }
}
}
/* End copied in from newer version of Font-Awesome 4.3.0 */
#overlay {
display: flex;
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background-color: rgba(0, 0, 0, 0.7);
align-items: center;
justify-content: center;
z-index: 999;
}
.spinner {
border: 6px solid rgba(255, 255, 255, 0.3);
border-radius: 50%;
border-top: 6px solid #3498db;
width: 50px;
height: 50px;
animation: spin 1s linear infinite;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}

View File

@ -132,7 +132,8 @@
{% if project_enable %}
<a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a>
{% endif %}
</div>
<a class="btn btn-default navbar-btn navbar-right" id="import_page" style="margin-right: 5px !important" id="import-cmdline-button" href="{% url 'cmdlines' %}">Import command line builds</a>
</div>
</div>
</nav>

View File

@ -0,0 +1,198 @@
{% extends "base.html" %}
{% load projecttags %}
{% load humanize %}
{% block title %} Import Builds from eventlogs - Toaster {% endblock %}
{% block pagecontent %}
<div class="container-fluid">
<div id="overlay" class="hide">
<div class="spinner">
<div class="fa-spin">
</div>
</div>
</div>
<div class="row">
<div class="col-md-12">
<div class="page-header">
<div class="row">
<div class="col-md-6">
<h1>Import command line builds</h1>
</div>
{% if import_all %}
<div class="col-md-6">
<button id="import_all" type="button" class="btn btn-primary navbar-btn navbar-right">
<span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import All
</button>
</div>
{% endif %}
</div>
</div>
{% if messages %}
<div class="row-fluid" id="empty-state-{{table_name}}">
{% for message in messages %}
<div class="alert alert-danger">{{message}}</div>
{%endfor%}
</div>
{% endif %}
<div class="row">
<h4 style="margin-left: 15px;"><strong>Import eventlog file</strong></h4>
<form method="POST" enctype="multipart/form-data" action="{% url 'cmdlines' %}" id="form_file">
{% csrf_token %}
<div class="col-md-6" style="padding-left: 20px;">
<div class="row">
<input type="hidden" value="{{dir}}" name="dir">
<div class="col-md-3"> {{ form.eventlog_file}} </div>
</div>
<div class="row" style="padding-top: 10px;">
<div class="col-md-6">
<button id="file_import" type="submit" disabled="disabled" class="btn btn-default navbar-btn" >
<span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import
</button>
</div>
</div>
</div>
</form>
</div>
<div class="row" style="padding-top: 20px;">
<div class="col-md-8 ">
<h4><strong>Eventlogs from existing build directory: </strong>
<a href="#" data-toggle="tooltip" title="{{dir}}">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle" viewBox="0 0 16 16" data-toggle="tooltip">
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14m0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16"/>
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0"/>
</svg>
</a>
</h4>
{% if files %}
<div class="table-responsive">
<table class="table col-md-6 table-bordered table-hover">
<thead>
<tr class="row">
<th scope="col">Name</th>
<th scope="col">Size</th>
<th scope="col">Action</th>
</tr>
</thead>
<tbody>
{% for file in files %}
<tr class="row" style="height: 48px;">
<th scope="row" class="col-md-4" style="vertical-align: middle;">
<input type="hidden" value="{{file.name}}" name="{{file.name}}">{{file.name}}
</th>
<td class="col-md-4 align-middle" style="vertical-align: middle;">{{file.size|filesizeformat}}</td>
<td class="col-md-4 align-middle" style="vertical-align: middle;">
{% if file.imported == True and file.build_id is not None %}
<a href="{% url 'builddashboard' file.build_id %}">Build Details</a>
{% elif request.session.file == file.name or request.session.all_builds %}
<a data-toggle="tooltip" title="Build in progress">
<span class="glyphicon glyphicon-upload" style="font-size: 18px; color:grey"></span>
</a>
{%else%}
<a onclick="_ajax_update('{{file.name}}', false, '{{dir}}')" data-toggle="tooltip" title="Import File">
<span class="glyphicon glyphicon-upload" style="font-size: 18px;"></span>
</a>
{%endif%}
</td>
</tr>
{% endfor%}
</tbody>
</table>
</div>
{% else %}
<div class="row-fluid" id="empty-state-{{table_name}}">
<div class="alert alert-info">Sorry - no files found</div>
</div>
{%endif%}
</div>
</div>
</div>
</div>
</div>
<script>
function _ajax_update(file, all, dir){
function getCookie(name) {
var cookieValue = null;
if (document.cookie && document.cookie !== '') {
var cookies = document.cookie.split(';');
for (var i = 0; i < cookies.length; i++) {
var cookie = jQuery.trim(cookies[i]);
// Does this cookie string begin with the name we want?
if (cookie.substring(0, name.length + 1) === (name + '=')) {
cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
break;
}
}
}
return cookieValue;
}
var csrftoken = getCookie('csrftoken');
function csrfSafeMethod(method) {
// these HTTP methods do not require CSRF protection
return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
}
$.ajaxSetup({
beforeSend: function (xhr, settings) {
if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
xhr.setRequestHeader("X-CSRFToken", csrftoken);
}
}
});
$.ajax({
url:'/toastergui/cmdline/',
type: "POST",
data: {file: file, all: all, dir: dir},
success:function(data){
window.location = '/toastergui/builds/'
},
complete:function(data){
},
error:function (xhr, textStatus, thrownError){
console.log('fail');
}
});
}
$('#import_all').on('click', function(){
_ajax_update("{{files | safe}}", true, "{{dir | safe}}");
});
$('#import_page').hide();
$(function () {
$('[data-toggle="tooltip"]').tooltip()
})
$("#id_eventlog_file").change(function(){
$('#file_import').prop("disabled", false);
$('#file_import').addClass('btn-primary')
$('#file_import').removeClass('btn-default')
})
$(document).ajaxStart(function(){
$('#overlay').removeClass('hide');
window.setTimeout(
function() {
window.location = '/toastergui/builds/'
}, 10000)
});
$( "#form_file").on( "submit", function( event ) {
$('#overlay').removeClass('hide');
window.setTimeout(
function() {
window.location = '/toastergui/builds/'
}, 10000)
});
</script>
{% endblock %}

View File

@ -15,7 +15,7 @@
<p>A web interface to <a href="https://www.openembedded.org">OpenEmbedded</a> and <a href="https://docs.yoctoproject.org/bitbake.html">BitBake</a>, the <a href="https://www.yoctoproject.org">Yocto Project</a> build system.</p>
<p class="top-air">
<a class="btn btn-info btn-lg" href="http://docs.yoctoproject.org/toaster-manual/setup-and-use.html#setting-up-and-using-toaster">
<a class="btn btn-info btn-lg" href="http://docs.yoctoproject.org/toaster-manual/setup-and-use.html#setting-up-and-using-toaster" style="min-width: 460px;">
Toaster is ready to capture your command line builds
</a>
</p>
@ -23,7 +23,7 @@
{% if lvs_nos %}
{% if project_enable %}
<p class="top-air">
<a class="btn btn-primary btn-lg" href="{% url 'newproject' %}">
<a class="btn btn-primary btn-lg" href="{% url 'newproject' %}" style="min-width: 460px;">
Create your first Toaster project to run manage builds
</a>
</p>
@ -42,6 +42,12 @@
</div>
{% endif %}
<p class="top-air">
<a class="btn btn-info btn-lg" href="{% url 'cmdlines' %}" style="min-width: 460px;">
Import command line event logs from build directory
</a>
</p>
<ul class="list-unstyled lead">
<li>
<a href="http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual">

View File

@ -95,6 +95,7 @@ urlpatterns = [
# project URLs
url(r'^newproject/$', views.newproject, name='newproject'),
url(r'^cmdline/$', views.CommandLineBuilds.as_view(), name='cmdlines'),
url(r'^projects/$',
tables.ProjectsTable.as_view(template_name="projects-toastertable.html"),
name='all-projects'),

View File

@ -6,24 +6,36 @@
# SPDX-License-Identifier: GPL-2.0-only
#
import ast
import re
import subprocess
import sys
import bb.cooker
from bb.ui import toasterui
from bb.ui import eventreplay
from django.db.models import F, Q, Sum
from django.db import IntegrityError
from django.shortcuts import render, redirect, get_object_or_404
from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect
from django.utils.http import urlencode
from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe
from orm.models import LogMessage, Variable, Package_Dependency, Package
from orm.models import Task_Dependency, Package_File
from orm.models import Target_Installed_Package, Target_File
from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File
from orm.models import BitbakeVersion, CustomImageRecipe
from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports
from django.urls import reverse, resolve
from django.contrib import messages
from django.core.exceptions import ObjectDoesNotExist
from django.core.files.storage import FileSystemStorage
from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponseNotFound, JsonResponse
from django.utils import timezone
from django.views.generic import TemplateView
from datetime import timedelta, datetime
from toastergui.templatetags.projecttags import json as jsonfilter
from decimal import Decimal
@ -32,6 +44,10 @@ import os
from os.path import dirname
import mimetypes
from toastergui.forms import LoadFileForm
from collections import namedtuple
import logging
from toastermain.logs import log_view_mixin
@ -41,6 +57,7 @@ logger = logging.getLogger("toaster")
# Project creation and managed build enable
project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
import_page = False
class MimeTypeFinder(object):
# setting this to False enables additional non-standard mimetypes
@ -1940,3 +1957,155 @@ if True:
except (ObjectDoesNotExist, IOError):
return toaster_render(request, "unavailable_artifact.html")
class CommandLineBuilds(TemplateView):
model = EventLogsImports
template_name = 'command_line_builds.html'
def get_context_data(self, **kwargs):
context = super(CommandLineBuilds, self).get_context_data(**kwargs)
#get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf
eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value'])
if eventlog:
logs_dir = os.path.dirname(eventlog.decode().strip('\n'))
files = os.listdir(logs_dir)
imported_files = EventLogsImports.objects.all()
files_list = []
# Filter files that end with ".json"
event_files = []
for file in files:
if file.endswith(".json"):
# because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog
with open("{}/{}".format(logs_dir, file)) as efile:
content = efile.read()
if 'allvariables' in content:
event_files.append(file)
#build dict for template using db data
for event_file in event_files:
if imported_files.filter(name=event_file):
files_list.append({
'name': event_file,
'imported': True,
'build_id': imported_files.filter(name=event_file)[0].build_id,
'size': os.path.getsize("{}/{}".format(logs_dir, event_file))
})
else:
files_list.append({
'name': event_file,
'imported': False,
'build_id': None,
'size': os.path.getsize("{}/{}".format(logs_dir, event_file))
})
context['import_all'] = True
context['files'] = files_list
context['dir'] = logs_dir
else:
context['files'] = []
context['dir'] = ''
# enable session variable
if not self.request.session.get('file'):
self.request.session['file'] = ""
context['form'] = LoadFileForm()
context['project_enable'] = project_enable
return context
def post(self, request, **kwargs):
logs_dir = request.POST.get('dir')
all_files = request.POST.get('all')
imported_files = EventLogsImports.objects.all()
try:
if all_files == 'true':
# use of session variable to deactivate icon for builds in progress
request.session['all_builds'] = True
request.session.modified = True
request.session.save()
files = ast.literal_eval(request.POST.get('file'))
for file in files:
if imported_files.filter(name=file.get('name')).exists():
imported_files.filter(name=file.get('name'))[0].imported = True
else:
with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile:
# load variables from the first line
variables = None
while line := eventfile.readline().strip():
try:
variables = json.loads(line)['allvariables']
break
except (KeyError, json.JSONDecodeError):
continue
if not variables:
raise Exception("File content missing build variables")
eventfile.seek(0)
params = namedtuple('ConfigParams', ['observe_only'])(True)
player = eventreplay.EventPlayer(eventfile, variables)
toasterui.main(player, player, params)
event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True)
event_log_import.build_id = Build.objects.last().id
event_log_import.save()
else:
if self.request.FILES.get('eventlog_file'):
file = self.request.FILES['eventlog_file']
else:
file = request.POST.get('file')
# use of session variable to deactivate icon for build in progress
request.session['file'] = file
request.session['all_builds'] = False
request.session.modified = True
request.session.save()
if imported_files.filter(name=file).exists():
imported_files.filter(name=file)[0].imported = True
else:
if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile):
variables = None
while line := file.readline().strip():
try:
variables = json.loads(line)['allvariables']
break
except (KeyError, json.JSONDecodeError):
continue
if not variables:
raise Exception("File content missing build variables")
file.seek(0)
params = namedtuple('ConfigParams', ['observe_only'])(True)
player = eventreplay.EventPlayer(file, variables)
if not os.path.exists('{}/{}'.format(logs_dir, file.name)):
fs = FileSystemStorage(location=logs_dir)
fs.save(file.name, file)
toasterui.main(player, player, params)
else:
with open("{}/{}".format(logs_dir, file)) as eventfile:
# load variables from the first line
variables = None
while line := eventfile.readline().strip():
try:
variables = json.loads(line)['allvariables']
break
except (KeyError, json.JSONDecodeError):
continue
if not variables:
raise Exception("File content missing build variables")
eventfile.seek(0)
params = namedtuple('ConfigParams', ['observe_only'])(True)
player = eventreplay.EventPlayer(eventfile, variables)
toasterui.main(player, player, params)
event_log_import = EventLogsImports.objects.create(name=file, imported=True)
event_log_import.build_id = Build.objects.last().id
event_log_import.save()
request.session['file'] = ""
except Exception:
messages.add_message(
self.request,
messages.ERROR,
"The file content is not in the correct format. Update file content or upload a different file."
)
return HttpResponseRedirect("/toastergui/cmdline/")
return HttpResponseRedirect('/toastergui/builds/')