From 41c867480f1e20bd3b168258ed71450499ea6ccf Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Tue, 20 Nov 2012 16:02:40 -0500 Subject: Removed 2-step parse for overheads. --- parse/ft.py | 129 +++++++++++++++++---------------------------------------- parse/sched.py | 1 - 2 files changed, 39 insertions(+), 91 deletions(-) (limited to 'parse') diff --git a/parse/ft.py b/parse/ft.py index cbf75f2..c5f1522 100644 --- a/parse/ft.py +++ b/parse/ft.py @@ -7,94 +7,68 @@ import subprocess from point import Measurement,Type -SPLIT_DATA_NAME = "overhead={}.bin" -FT_DATA_NAME = "sorted-ft.bin" -FIELDS = ["Overhead", "samples", "max", "avg", "min", "med", "std", "var"] +FT_SPLIT_NAME = "overhead={}.bin" +FT_SORTED_NAME = "sorted-ft.bin" +FT_ERR_NAME = "err-ft" -def get_ft_output(data_dir, cycles, out_dir, force=False): - """ - Create and return file containing analyzed overhead data - """ +def extract_ft_data(result, data_dir, cycles, tmp_dir): freg = conf.FILES['ft_data'] + "$" bins = [f for f in os.listdir(data_dir) if re.match(freg, f)] - output_file = "{}/out-ft".format(out_dir) + if not len(bins): + return False - if os.path.isfile(output_file): - if force: - os.remove(output_file) - else: - return output_file + bin_file = "{}/{}".format(data_dir, bins[0]) - if len(bins) != 0: - bin_file = "{}/{}".format(data_dir, bins[0]) - err_file = open("%s/err-ft" % out_dir, 'w') + with open("%s/%s" % (tmp_dir, FT_ERR_NAME), 'w') as err_file: + sorted_bin = sort_ft(bin_file, err_file, tmp_dir) - sorted_bin = sort_ft(bin_file, err_file, out_dir) - make_data_file(sorted_bin, cycles, output_file, err_file, out_dir) + for event in conf.BASE_EVENTS: + parse_overhead(result, sorted_bin, event, cycles, + tmp_dir, err_file) os.remove(sorted_bin) - return output_file - else: - return None - return output_file + return True -def fmt_cell(x): - if type(x) == str: - return "%15s" % x - if type(x) == int: - return "%15d" % x - else: - return "%15.3f" % x +def parse_overhead(result, overhead_bin, overhead, cycles, out_dir, err_file): + ovh_fname = "{}/{}".format(out_dir, FT_SPLIT_NAME).format(overhead) -def make_data_file(sorted_bin, cycles, out_fname, err_file, out_dir): - """ - Create file containing all overhead information. - """ - base_name = "{}/{}".format(out_dir, SPLIT_DATA_NAME) + if os.path.exists(ovh_fname): + os.remove(ovh_fname) + ovh_file = open(ovh_fname, 'w') - with open(out_fname, "w") as f: - f.write("#%s" % ", ".join(fmt_cell(x) for x in FIELDS)) - f.write("\n") + # Extract matching overhead events into a seperate file + cmd = [conf.BINS["split"], "-r", "-b", overhead, overhead_bin] + ret = subprocess.call(cmd, cwd=out_dir, stderr=err_file, stdout=ovh_file) + size = os.stat(ovh_fname).st_size - for event in conf.BASE_EVENTS: - ovh_fname = base_name.format(event.replace("_", "-")) - - if os.path.exists(ovh_fname): - os.remove(ovh_fname) - ovh_file = open(ovh_fname, 'w') - - # Extract matching overhead events into a seperate file - cmd = [conf.BINS["split"], "-r", "-b", event, sorted_bin] - ret = subprocess.call(cmd, cwd=out_dir, - stderr=err_file, stdout=ovh_file) - size = os.stat(ovh_fname).st_size + if ret: + raise Exception("Failed (%d) with command: %s" % (ret, " ".join(cmd))) + if not size: + os.remove(ovh_fname) - if ret: - err_file.write("Failed with command: %s" % " ".join(cmd)) - if not size: - os.remove(ovh_fname) - if not size or ret: - continue + if size and not ret: + # Map and sort file for stats + data = np.memmap(ovh_fname, dtype="float32", mode='c') + data /= float(cycles) # Scale for processor speed + data.sort() - # Map and sort file for stats - data = np.memmap(ovh_fname, dtype="float32", mode='c') - data /= float(cycles) # Scale for processor speed - data.sort() + m = Measurement("%s-%s" % (overhead_bin, overhead)) + m[Type.Max] = data[-1] + m[Type.Avg] = np.mean(data) + m[Type.Min] = data[0] + m[Type.Var] = np.var(data) - stats = [event, len(data), data[-1], np.mean(data), data[0], - np.median(data), np.std(data, ddof=1), np.var(data)] - f.write(", ".join([fmt_cell(x) for x in stats])) - f.write("\n") + result[overhead] = m - os.remove(ovh_fname) + os.remove(ovh_fname) def sort_ft(ft_file, err_file, out_dir): """ Create and return file with sorted overheads from @ft_file. """ - out_fname = "{}/{}".format(out_dir, FT_DATA_NAME) + out_fname = "{}/{}".format(out_dir, FT_SORTED_NAME) # Sort happens in-place sh.copyfile(ft_file, out_fname) @@ -105,28 +79,3 @@ def sort_ft(ft_file, err_file, out_dir): raise Exception("Sort failed with command: %s" % " ".join(cmd)) return out_fname - -def extract_ft_data(data_file, result, overheads): - """ - Return exp point with overhead measurements from data_file - """ - with open(data_file) as f: - data = f.read() - - for ovh in overheads: - regex = r"({}[^\n]*)".format(ovh) - line = re.search(regex, data) - - if not line: - continue - - vals = re.split(r"[,\s]+", line.groups(1)[0]) - - measure = Measurement("%s-%s" % (data_file, ovh)) - measure[Type.Max] = float(vals[FIELDS.index("max")]) - measure[Type.Avg] = float(vals[FIELDS.index("avg")]) - measure[Type.Var] = float(vals[FIELDS.index("var")]) - - result[ovh] = measure - - return result diff --git a/parse/sched.py b/parse/sched.py index bbf6e10..3e30880 100644 --- a/parse/sched.py +++ b/parse/sched.py @@ -11,7 +11,6 @@ import os import re import numpy as np import subprocess -import pprint from collections import namedtuple,defaultdict from operator import methodcaller -- cgit v1.2.2