From 16dfc8df20c6befeed423217a2e0f5ae59b5a04d Mon Sep 17 00:00:00 2001 From: Jonathan Herman Date: Mon, 18 Mar 2013 13:15:42 -0400 Subject: Use smarter defaults which can handle data from Bjorns scripts. --- common.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ config/config.py | 5 +++-- gen/generator.py | 5 ++--- parse/dir_map.py | 3 +++ parse/ft.py | 2 +- parse/point.py | 3 +-- parse_exps.py | 35 ++++++++++++++++++------------ run/executable/executable.py | 2 +- run/litmus_util.py | 50 ------------------------------------------- run/tracer.py | 10 ++++----- run_exps.py | 9 ++++---- 11 files changed, 92 insertions(+), 83 deletions(-) diff --git a/common.py b/common.py index d080e1a..cba31bf 100644 --- a/common.py +++ b/common.py @@ -1,5 +1,6 @@ import os import re +import stat import subprocess import sys @@ -134,3 +135,53 @@ def load_params(fname): raise IOError("Invalid param file: %s\n%s" % (fname, e)) return params + + +def num_cpus(): + '''Return the number of CPUs in the system.''' + + lnx_re = re.compile(r'^(processor|online)') + cpus = 0 + + with open('/proc/cpuinfo', 'r') as f: + for line in f: + if lnx_re.match(line): + cpus += 1 + return cpus + +def ft_freq(): + umachine = subprocess.check_output(["uname", "-m"]) + + if re.match("armv7", umachine): + # Arm V7s use a millisecond timer + freq = 1000.0 + elif re.match("x86", umachine): + # X86 timer is equal to processor clock + reg = re.compile(r'^cpu MHz\s*:\s*(?P\d+)', re.M) + with open('/proc/cpuinfo', 'r') as f: + data = f.read() + + match = re.search(reg, data) + if not match: + raise Exception("Cannot parse CPU frequency from x86 CPU!") + freq = int(match.group('FREQ')) + else: + # You're on your own + freq = 0 + return freq + + +def uname_matches(reg): + data = subprocess.check_output(["uname", "-r"]) + return bool( re.match(reg, data) ) + +def is_executable(fname): + '''Return whether the file passed in is executable''' + mode = os.stat(fname)[stat.ST_MODE] + return mode & stat.S_IXUSR and mode & stat.S_IRUSR + +def is_device(dev): + if not os.path.exists(dev): + return False + mode = os.stat(dev)[stat.ST_MODE] + return not (not mode & stat.S_IFCHR) diff --git a/config/config.py b/config/config.py index 1199278..a6edece 100644 --- a/config/config.py +++ b/config/config.py @@ -1,6 +1,6 @@ from __future__ import print_function import itertools -from common import get_executable +from common import get_executable,ft_freq '''Paths to binaries.''' BINS = {'rtspin' : get_executable('rtspin', 'liblitmus'), @@ -16,6 +16,7 @@ BINS = {'rtspin' : get_executable('rtspin', 'liblitmus'), '''Names of output files.''' FILES = {'ft_data' : 'ft.bin', + 'ft_matches' : r'(ft.*\.bin)|(.*\.ft)', 'linux_data' : 'trace.dat', 'sched_data' : 'st-{}.bin', 'log_data' : 'trace.slog'} @@ -34,7 +35,7 @@ DEFAULTS = {'params_file' : 'params.py', 'sched_file' : 'sched.py', 'duration' : 10, 'spin' : 'rtspin', - 'cycles' : 2000} + 'cycles' : ft_freq() or 2000} '''Default sched_trace events (this is all of them).''' SCHED_EVENTS = range(501, 513) diff --git a/gen/generator.py b/gen/generator.py index dbf2a7b..8c3048b 100644 --- a/gen/generator.py +++ b/gen/generator.py @@ -1,11 +1,10 @@ import gen.rv as rv import os -import run.litmus_util as lu import shutil as sh from Cheetah.Template import Template from collections import namedtuple -from common import get_config_option +from common import get_config_option,num_cpus from config.config import DEFAULTS,PARAMS from gen.dp import DesignPointGenerator from parse.col_map import ColMapBuilder @@ -69,7 +68,7 @@ class Generator(object): if 'cpus' in params: cpus = min(map(int, params['cpus'])) else: - cpus = lu.num_cpus() + cpus = num_cpus() try: config = get_config_option("RELEASE_MASTER") and True except: diff --git a/parse/dir_map.py b/parse/dir_map.py index 601dd3b..a8d2a83 100644 --- a/parse/dir_map.py +++ b/parse/dir_map.py @@ -56,6 +56,9 @@ class DirMap(object): remove_childless2(self.root) + def is_empty(self): + return not len(self.root.children) + def write(self, out_dir): def write2(path, node): out_path = "/".join(path) diff --git a/parse/ft.py b/parse/ft.py index 19453d1..98405f4 100644 --- a/parse/ft.py +++ b/parse/ft.py @@ -63,7 +63,7 @@ def extract_ft_data(result, data_dir, work_dir, cycles): data_dir = os.path.abspath(data_dir) work_dir = os.path.abspath(work_dir) - freg = conf.FILES['ft_data'] + "$" + freg = conf.FILES['ft_matches'] + "$" bins = [f for f in os.listdir(data_dir) if re.match(freg, f)] if not len(bins): diff --git a/parse/point.py b/parse/point.py index d577306..f2b266a 100644 --- a/parse/point.py +++ b/parse/point.py @@ -128,8 +128,7 @@ class ExpPoint(object): self.stats[type] = value def __str__(self): - # return "\n%s" % (self.id, dict_str(self.stats)) - return "" % (self.id) + return "\n%s" % (self.id, dict_str(self.stats)) def get_stats(self): return self.stats.keys() diff --git a/parse_exps.py b/parse_exps.py index 7dfc9cd..8fcf5c3 100755 --- a/parse_exps.py +++ b/parse_exps.py @@ -22,8 +22,6 @@ def parse_args(): # TODO: convert data-dir to proper option, clean 'dest' options parser = OptionParser("usage: %prog [options] [data_dir]...") - print("default to no params.py") - parser.add_option('-o', '--out', dest='out', help='file or directory for data output', default='parse-data') parser.add_option('-c', '--clean', action='store_true', default=False, @@ -44,15 +42,15 @@ ExpData = namedtuple('ExpData', ['path', 'params', 'work_dir']) def get_exp_params(data_dir, cm_builder): param_file = "%s/%s" % (data_dir, conf.DEFAULTS['params_file']) - if not os.path.isfile: - raise Exception("No param file '%s' exists!" % param_file) - - params = load_params(param_file) + if os.path.isfile(param_file): + params = load_params(param_file) - # Store parameters in cm_builder, which will track which parameters change - # across experiments - for key, value in params.iteritems(): - cm_builder.try_add(key, value) + # Store parameters in cm_builder, which will track which parameters change + # across experiments + for key, value in params.iteritems(): + cm_builder.try_add(key, value) + else: + params = {} # Cycles must be present for feather-trace measurement parsing if conf.PARAMS['cycles'] not in params: @@ -164,16 +162,25 @@ def main(): if opts.force and os.path.exists(opts.out): sh.rmtree(opts.out) - result_table = result_table.reduce() + reduced_table = result_table.reduce() sys.stderr.write("Writing result...\n") if opts.write_map: # Write summarized results into map - result_table.write_map(opts.out) + reduced_table.write_map(opts.out) else: # Write out csv directories for all variable params - dir_map = result_table.to_dir_map() - dir_map.write(opts.out) + dir_map = reduced_table.to_dir_map() + + # No csvs to write, assume user meant to print out data + if dir_map.is_empty(): + sys.stderr.write("Too little data to make csv files.\n") + if not opts.verbose: + for key, exp in result_table: + for e in exp: + print(e) + else: + dir_map.write(opts.out) if __name__ == '__main__': main() diff --git a/run/executable/executable.py b/run/executable/executable.py index 0a408b7..02e35ae 100644 --- a/run/executable/executable.py +++ b/run/executable/executable.py @@ -1,7 +1,7 @@ import sys import subprocess import signal -from ..litmus_util import is_executable +from common import is_executable class Executable(object): '''Parent object that represents an executable for use in task-sets.''' diff --git a/run/litmus_util.py b/run/litmus_util.py index 8a7f87d..b9080c1 100644 --- a/run/litmus_util.py +++ b/run/litmus_util.py @@ -1,43 +1,8 @@ import re import time import subprocess -import os -import stat import config.config as conf -def num_cpus(): - '''Return the number of CPUs in the system.''' - - lnx_re = re.compile(r'^(processor|online)') - cpus = 0 - - with open('/proc/cpuinfo', 'r') as f: - for line in f: - if lnx_re.match(line): - cpus += 1 - return cpus - -def ft_freq(): - umachine = subprocess.check_output(["uname", "-m"]) - - if re.match("armv7", umachine): - # Arm V7s use a millisecond timer - freq = 1000.0 - elif re.match("x86", umachine): - # X86 timer is equal to processor clock - reg = re.compile(r'^cpu MHz\s*:\s*(?P\d+)', re.M) - with open('/proc/cpuinfo', 'r') as f: - data = f.read() - - match = re.search(reg, data) - if not match: - raise Exception("Cannot parse CPU frequency from x86 CPU!") - freq = int(match.group('FREQ')) - else: - # You're on your own - freq = 0 - return freq - def switch_scheduler(switch_to_in): '''Switch the scheduler to whatever is passed in. @@ -60,21 +25,6 @@ def switch_scheduler(switch_to_in): if switch_to != cur_plugin: raise Exception("Could not switch to plugin: %s" % switch_to) -def uname_matches(reg): - data = subprocess.check_output(["uname", "-r"]) - return bool( re.match(reg, data) ) - -def is_executable(fname): - '''Return whether the file passed in is executable''' - mode = os.stat(fname)[stat.ST_MODE] - return mode & stat.S_IXUSR and mode & stat.S_IRUSR - -def is_device(dev): - if not os.path.exists(dev): - return False - mode = os.stat(dev)[stat.ST_MODE] - return not (not mode & stat.S_IFCHR) - def waiting_tasks(): reg = re.compile(r'^ready.*?(?P\d+)$', re.M) with open('/proc/litmus/stats', 'r') as f: diff --git a/run/tracer.py b/run/tracer.py index 303aae0..065797c 100644 --- a/run/tracer.py +++ b/run/tracer.py @@ -1,7 +1,7 @@ -from . import litmus_util import os import config.config as conf +from common import is_device,num_cpus from operator import methodcaller from run.executable.ftcat import FTcat,Executable @@ -58,7 +58,7 @@ class LogTracer(Tracer): @staticmethod def enabled(): - return litmus_util.is_device(LogTracer.DEVICE_STR) + return is_device(LogTracer.DEVICE_STR) def stop_tracing(self): map(methodcaller('interrupt'), self.bins) @@ -71,7 +71,7 @@ class SchedTracer(Tracer): super(SchedTracer, self).__init__("Sched Trace", output_dir) if SchedTracer.enabled(): - for cpu in range(litmus_util.num_cpus()): + for cpu in range(num_cpus()): # Executable will close the stdout/stderr files stdout_f = open('%s/st-%d.bin' % (self.output_dir, cpu), 'w') stderr_f = open('%s/st-%d-stderr.txt' % (self.output_dir, cpu), 'w') @@ -83,7 +83,7 @@ class SchedTracer(Tracer): @staticmethod def enabled(): - return litmus_util.is_device("%s%d" % (SchedTracer.DEVICE_STR, 0)) + return is_device("%s%d" % (SchedTracer.DEVICE_STR, 0)) class OverheadTracer(Tracer): DEVICE_STR = '/dev/litmus/ft_trace0' @@ -100,7 +100,7 @@ class OverheadTracer(Tracer): @staticmethod def enabled(): - return litmus_util.is_device(OverheadTracer.DEVICE_STR) + return is_device(OverheadTracer.DEVICE_STR) class PerfTracer(Tracer): def __init__(self, output_dir): diff --git a/run_exps.py b/run_exps.py index 195d3f8..a05ff93 100755 --- a/run_exps.py +++ b/run_exps.py @@ -2,13 +2,12 @@ from __future__ import print_function import config.config as conf -import run.litmus_util as lu import os import re import shutil import traceback -from common import load_params,get_executable +from common import load_params,get_executable,uname_matches,ft_freq from optparse import OptionParser from run.executable.executable import Executable from run.experiment import Experiment,ExperimentDone @@ -125,7 +124,7 @@ def load_experiment(sched_file, scheduler, duration, param_file, out_dir): (conf.PARAMS['dur'], duration)]) # Feather-trace clock frequency saved for accurate overhead parsing - ft_freq = lu.ft_freq() + ft_freq = ft_freq() if ft_freq: out_params[conf.PARAMS['cycles']] = ft_freq @@ -146,7 +145,7 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): proc_entries = [] executables = [] - if kernel and not lu.uname_matches(kernel): + if kernel and not uname_matches(kernel): raise InvalidKernel(kernel) # Parse values for proc entries @@ -178,7 +177,7 @@ def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): if re.match(".*spin", real_spin): real_args = ['-w'] + real_args + [duration] - if not lu.is_executable(real_spin): + if not is_executable(real_spin): raise OSError("Cannot run spin %s: %s" % (real_spin, name)) executables += [Executable(real_spin, real_args)] -- cgit v1.2.2