mirror of
git://git.yoctoproject.org/poky.git
synced 2025-07-19 21:09:03 +02:00
ffae400179
This adds SPDX license headers in place of the wide assortment of things currently in our script headers. We default to GPL-2.0-only except for the oeqa code where it was clearly submitted and marked as MIT on the most part or some scripts which had the "or later" GPL versioning. The patch also drops other obsolete bits of file headers where they were encoountered such as editor modelines, obsolete maintainer information or the phrase "All rights reserved" which is now obsolete and not required in copyright headers (in this case its actually confusing for licensing as all rights were not reserved). More work is needed for OE-Core but this takes care of the bulk of the scripts and meta/lib directories. The top level LICENSE files are tweaked to match the new structure and the SPDX naming. (From OE-Core rev: f8c9c511b5f1b7dbd45b77f345cb6c048ae6763e) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
301 lines
11 KiB
Python
Executable File
301 lines
11 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# Script for comparing buildstats from two different builds
|
|
#
|
|
# Copyright (c) 2016, Intel Corporation.
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import argparse
|
|
import glob
|
|
import logging
|
|
import math
|
|
import os
|
|
import sys
|
|
from operator import attrgetter
|
|
|
|
# Import oe libs
|
|
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
|
sys.path.append(os.path.join(scripts_path, 'lib'))
|
|
from buildstats import BuildStats, diff_buildstats, taskdiff_fields, BSVerDiff
|
|
|
|
|
|
# Setup logging
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
|
log = logging.getLogger()
|
|
|
|
|
|
class ScriptError(Exception):
|
|
"""Exception for internal error handling of this script"""
|
|
pass
|
|
|
|
|
|
def read_buildstats(path, multi):
|
|
"""Read buildstats"""
|
|
if not os.path.exists(path):
|
|
raise ScriptError("No such file or directory: {}".format(path))
|
|
|
|
if os.path.isfile(path):
|
|
return BuildStats.from_file_json(path)
|
|
|
|
if os.path.isfile(os.path.join(path, 'build_stats')):
|
|
return BuildStats.from_dir(path)
|
|
|
|
# Handle a non-buildstat directory
|
|
subpaths = sorted(glob.glob(path + '/*'))
|
|
if len(subpaths) > 1:
|
|
if multi:
|
|
log.info("Averaging over {} buildstats from {}".format(
|
|
len(subpaths), path))
|
|
else:
|
|
raise ScriptError("Multiple buildstats found in '{}'. Please give "
|
|
"a single buildstat directory of use the --multi "
|
|
"option".format(path))
|
|
bs = None
|
|
for subpath in subpaths:
|
|
if os.path.isfile(subpath):
|
|
_bs = BuildStats.from_file_json(subpath)
|
|
else:
|
|
_bs = BuildStats.from_dir(subpath)
|
|
if bs is None:
|
|
bs = _bs
|
|
else:
|
|
bs.aggregate(_bs)
|
|
if not bs:
|
|
raise ScriptError("No buildstats found under {}".format(path))
|
|
|
|
return bs
|
|
|
|
|
|
def print_ver_diff(bs1, bs2):
|
|
"""Print package version differences"""
|
|
|
|
diff = BSVerDiff(bs1, bs2)
|
|
|
|
maxlen = max([len(r) for r in set(bs1.keys()).union(set(bs2.keys()))])
|
|
fmt_str = " {:{maxlen}} ({})"
|
|
|
|
if diff.new:
|
|
print("\nNEW RECIPES:")
|
|
print("------------")
|
|
for name, val in sorted(diff.new.items()):
|
|
print(fmt_str.format(name, val.nevr, maxlen=maxlen))
|
|
|
|
if diff.dropped:
|
|
print("\nDROPPED RECIPES:")
|
|
print("----------------")
|
|
for name, val in sorted(diff.dropped.items()):
|
|
print(fmt_str.format(name, val.nevr, maxlen=maxlen))
|
|
|
|
fmt_str = " {0:{maxlen}} {1:<20} ({2})"
|
|
if diff.rchanged:
|
|
print("\nREVISION CHANGED:")
|
|
print("-----------------")
|
|
for name, val in sorted(diff.rchanged.items()):
|
|
field1 = "{} -> {}".format(val.left.revision, val.right.revision)
|
|
field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
|
|
print(fmt_str.format(name, field1, field2, maxlen=maxlen))
|
|
|
|
if diff.vchanged:
|
|
print("\nVERSION CHANGED:")
|
|
print("----------------")
|
|
for name, val in sorted(diff.vchanged.items()):
|
|
field1 = "{} -> {}".format(val.left.version, val.right.version)
|
|
field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
|
|
print(fmt_str.format(name, field1, field2, maxlen=maxlen))
|
|
|
|
if diff.echanged:
|
|
print("\nEPOCH CHANGED:")
|
|
print("--------------")
|
|
for name, val in sorted(diff.echanged.items()):
|
|
field1 = "{} -> {}".format(val.left.epoch, val.right.epoch)
|
|
field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
|
|
print(fmt_str.format(name, field1, field2, maxlen=maxlen))
|
|
|
|
|
|
def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',)):
|
|
"""Diff task execution times"""
|
|
def val_to_str(val, human_readable=False):
|
|
"""Convert raw value to printable string"""
|
|
def hms_time(secs):
|
|
"""Get time in human-readable HH:MM:SS format"""
|
|
h = int(secs / 3600)
|
|
m = int((secs % 3600) / 60)
|
|
s = secs % 60
|
|
if h == 0:
|
|
return "{:02d}:{:04.1f}".format(m, s)
|
|
else:
|
|
return "{:d}:{:02d}:{:04.1f}".format(h, m, s)
|
|
|
|
if 'time' in val_type:
|
|
if human_readable:
|
|
return hms_time(val)
|
|
else:
|
|
return "{:.1f}s".format(val)
|
|
elif 'bytes' in val_type and human_readable:
|
|
prefix = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi']
|
|
dec = int(math.log(val, 2) / 10)
|
|
prec = 1 if dec > 0 else 0
|
|
return "{:.{prec}f}{}B".format(val / (2 ** (10 * dec)),
|
|
prefix[dec], prec=prec)
|
|
elif 'ops' in val_type and human_readable:
|
|
prefix = ['', 'k', 'M', 'G', 'T', 'P']
|
|
dec = int(math.log(val, 1000))
|
|
prec = 1 if dec > 0 else 0
|
|
return "{:.{prec}f}{}ops".format(val / (1000 ** dec),
|
|
prefix[dec], prec=prec)
|
|
return str(int(val))
|
|
|
|
def sum_vals(buildstats):
|
|
"""Get cumulative sum of all tasks"""
|
|
total = 0.0
|
|
for recipe_data in buildstats.values():
|
|
for bs_task in recipe_data.tasks.values():
|
|
total += getattr(bs_task, val_type)
|
|
return total
|
|
|
|
if min_val:
|
|
print("Ignoring tasks less than {} ({})".format(
|
|
val_to_str(min_val, True), val_to_str(min_val)))
|
|
if min_absdiff:
|
|
print("Ignoring differences less than {} ({})".format(
|
|
val_to_str(min_absdiff, True), val_to_str(min_absdiff)))
|
|
|
|
# Prepare the data
|
|
tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff)
|
|
|
|
# Sort our list
|
|
for field in reversed(sort_by):
|
|
if field.startswith('-'):
|
|
field = field[1:]
|
|
reverse = True
|
|
else:
|
|
reverse = False
|
|
tasks_diff = sorted(tasks_diff, key=attrgetter(field), reverse=reverse)
|
|
|
|
linedata = [(' ', 'PKG', ' ', 'TASK', 'ABSDIFF', 'RELDIFF',
|
|
val_type.upper() + '1', val_type.upper() + '2')]
|
|
field_lens = dict([('len_{}'.format(i), len(f)) for i, f in enumerate(linedata[0])])
|
|
|
|
# Prepare fields in string format and measure field lengths
|
|
for diff in tasks_diff:
|
|
task_prefix = diff.task_op if diff.pkg_op == ' ' else ' '
|
|
linedata.append((diff.pkg_op, diff.pkg, task_prefix, diff.task,
|
|
val_to_str(diff.absdiff),
|
|
'{:+.1f}%'.format(diff.reldiff),
|
|
val_to_str(diff.value1),
|
|
val_to_str(diff.value2)))
|
|
for i, field in enumerate(linedata[-1]):
|
|
key = 'len_{}'.format(i)
|
|
if len(field) > field_lens[key]:
|
|
field_lens[key] = len(field)
|
|
|
|
# Print data
|
|
print()
|
|
for fields in linedata:
|
|
print("{:{len_0}}{:{len_1}} {:{len_2}}{:{len_3}} {:>{len_4}} {:>{len_5}} {:>{len_6}} -> {:{len_7}}".format(
|
|
*fields, **field_lens))
|
|
|
|
# Print summary of the diffs
|
|
total1 = sum_vals(bs1)
|
|
total2 = sum_vals(bs2)
|
|
print("\nCumulative {}:".format(val_type))
|
|
print (" {} {:+.1f}% {} ({}) -> {} ({})".format(
|
|
val_to_str(total2 - total1), 100 * (total2-total1) / total1,
|
|
val_to_str(total1, True), val_to_str(total1),
|
|
val_to_str(total2, True), val_to_str(total2)))
|
|
|
|
|
|
def parse_args(argv):
|
|
"""Parse cmdline arguments"""
|
|
description="""
|
|
Script for comparing buildstats of two separate builds."""
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
description=description)
|
|
|
|
min_val_defaults = {'cputime': 3.0,
|
|
'read_bytes': 524288,
|
|
'write_bytes': 524288,
|
|
'read_ops': 500,
|
|
'write_ops': 500,
|
|
'walltime': 5}
|
|
min_absdiff_defaults = {'cputime': 1.0,
|
|
'read_bytes': 131072,
|
|
'write_bytes': 131072,
|
|
'read_ops': 50,
|
|
'write_ops': 50,
|
|
'walltime': 2}
|
|
|
|
parser.add_argument('--debug', '-d', action='store_true',
|
|
help="Verbose logging")
|
|
parser.add_argument('--ver-diff', action='store_true',
|
|
help="Show package version differences and exit")
|
|
parser.add_argument('--diff-attr', default='cputime',
|
|
choices=min_val_defaults.keys(),
|
|
help="Buildstat attribute which to compare")
|
|
parser.add_argument('--min-val', default=min_val_defaults, type=float,
|
|
help="Filter out tasks less than MIN_VAL. "
|
|
"Default depends on --diff-attr.")
|
|
parser.add_argument('--min-absdiff', default=min_absdiff_defaults, type=float,
|
|
help="Filter out tasks whose difference is less than "
|
|
"MIN_ABSDIFF, Default depends on --diff-attr.")
|
|
parser.add_argument('--sort-by', default='absdiff',
|
|
help="Comma-separated list of field sort order. "
|
|
"Prepend the field name with '-' for reversed sort. "
|
|
"Available fields are: {}".format(', '.join(taskdiff_fields)))
|
|
parser.add_argument('--multi', action='store_true',
|
|
help="Read all buildstats from the given paths and "
|
|
"average over them")
|
|
parser.add_argument('buildstats1', metavar='BUILDSTATS1', help="'Left' buildstat")
|
|
parser.add_argument('buildstats2', metavar='BUILDSTATS2', help="'Right' buildstat")
|
|
|
|
args = parser.parse_args(argv)
|
|
|
|
# We do not nedd/want to read all buildstats if we just want to look at the
|
|
# package versions
|
|
if args.ver_diff:
|
|
args.multi = False
|
|
|
|
# Handle defaults for the filter arguments
|
|
if args.min_val is min_val_defaults:
|
|
args.min_val = min_val_defaults[args.diff_attr]
|
|
if args.min_absdiff is min_absdiff_defaults:
|
|
args.min_absdiff = min_absdiff_defaults[args.diff_attr]
|
|
|
|
return args
|
|
|
|
|
|
def main(argv=None):
|
|
"""Script entry point"""
|
|
args = parse_args(argv)
|
|
if args.debug:
|
|
log.setLevel(logging.DEBUG)
|
|
|
|
# Validate sort fields
|
|
sort_by = []
|
|
for field in args.sort_by.split(','):
|
|
if field.lstrip('-') not in taskdiff_fields:
|
|
log.error("Invalid sort field '%s' (must be one of: %s)" %
|
|
(field, ', '.join(taskdiff_fields)))
|
|
sys.exit(1)
|
|
sort_by.append(field)
|
|
|
|
try:
|
|
bs1 = read_buildstats(args.buildstats1, args.multi)
|
|
bs2 = read_buildstats(args.buildstats2, args.multi)
|
|
|
|
if args.ver_diff:
|
|
print_ver_diff(bs1, bs2)
|
|
else:
|
|
print_task_diff(bs1, bs2, args.diff_attr, args.min_val,
|
|
args.min_absdiff, sort_by)
|
|
except ScriptError as err:
|
|
log.error(str(err))
|
|
return 1
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|