2954d78759
Rewrite the scripts that gather the metrics to be more generic. Extract the metrics repository cloning out so that we don't have to repeatedly clone it. Make the scripts parse their arguments using getopt and be more specific about what they're passed. In particular, this means that for the patch review run we pass the _repository_ that we're scanning so we can do git operations on it, and the base of the _layers_ (either a layer, or a directory containing layers) so we know what to scan. Be more clever when identifying what commits we need to analyse for patch review: instead of iterating through a set randomly, we can keep the revision list sorted and the checkout operations are a lot faster. Remove the commit/file count metric addition as patchreview itself does that now. Add an explicit --push option so it's easy to test the scripts in isolation without pushing. Signed-off-by: Ross Burton <ross.burton@arm.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
2.2 KiB
Executable File
#!/usr/bin/env python3 import json, os.path, collections import sys import argparse import subprocess import tempfile from datetime import datetime, date, timedelta
args = argparse.ArgumentParser(description="Generate CVE count data files") args.add_argument("-j", "--json", help="JSON data file to use") args.add_argument("-r", "--resultsdir", help="results directory to parse") args = args.parse_args()
try: with open(args.json) as f: counts = json.load(f) except FileNotFoundError: # if the file does not exist, start with an empty database. counts = {}
lastyear = {}
Write CVE counts by day
def round_to_day(val): return int((datetime.fromtimestamp(int(val)).date() - date(1970, 1, 1)).total_seconds())
a_year_ago = (datetime.now() - timedelta(days=365) - datetime(1970, 1, 1)).total_seconds()
for branch in os.listdir(args.resultsdir): branchdir = os.path.join(args.resultsdir, branch) for f in os.listdir(branchdir): ts = f.split(".")[0] rounded_ts = str(round_to_day(ts)) if rounded_ts not in counts: counts[rounded_ts] = {} if branch not in counts[rounded_ts]: cvereport = os.path.join(branchdir, f) with open(cvereport) as report: reportdata = json.load(report) count = 0 seen = [] for package in reportdata['package']: if branch in ['dunfell', 'kirkstone', 'langdale'] and package['name'] in ['linux-yocto']: continue for issue in package['issue']: if issue['status'] == "Unpatched" and issue['id'] not in seen: count = count + 1 seen.append(issue['id']) print("Adding count %s for branch %s from file %s (ts %s)" % (count, branch, cvereport, rounded_ts)) counts[rounded_ts][branch] = str(count)
for c in counts: if int(c) > a_year_ago: lastyear[c] = counts[c]
with open(args.json, "w") as f: json.dump(counts, f, sort_keys=True, indent="\t")
with open(args.json.replace(".json", "-lastyear.json") , "w") as f: json.dump(lastyear, f, sort_keys=True, indent="\t")