Improved error handling in parse_ and plot_exps.py.

8 files changed, 41 insertions, 19 deletions

diff --git a/parse/dir_map.py b/parse/dir_map.py
index a8d2a83..231d21a 100644
--- a/parse/dir_map.py
+++ b/parse/dir_map.py
@@ -96,7 +96,10 @@ class DirMap(object):
                     return
 
                 with open(path, 'rb') as f:
-                    data = np.loadtxt(f, delimiter=",")
+                    try:
+                        data = np.loadtxt(f, delimiter=",")
+                    except Exception as e:
+                        raise IOError("Cannot load '%s': %s" % (path, e.message))
 
                 # Convert to tuples of ints if possible, else floats
                 values = [map(lambda a:a if a%1 else int(a), t) for t in data]
diff --git a/parse/ft.py b/parse/ft.py
index 98405f4..1f05323 100644
--- a/parse/ft.py
+++ b/parse/ft.py
@@ -71,7 +71,6 @@ def extract_ft_data(result, data_dir, work_dir, cycles):
 
     bin_file = "{}/{}".format(data_dir, bins[0])
     if not os.path.getsize(bin_file):
-        sys.stderr.write("Empty feather trace file %s!" % bin_file)
         return False
 
     with open("%s/%s" % (work_dir, FT_ERR_NAME), 'w') as err_file:
diff --git a/parse/point.py b/parse/point.py
index f2b266a..ac47c70 100644
--- a/parse/point.py
+++ b/parse/point.py
@@ -8,8 +8,8 @@ from enum import Enum
 from collections import defaultdict
 
 Type = Enum(['Min','Max','Avg','Var'])
-default_typemap = {Type.Max : {Type.Max : 1, Type.Min : 1, Type.Avg : 1, Type.Var : 1},
-                   Type.Min : {Type.Max : 1, Type.Min : 1, Type.Avg : 1, Type.Var : 1},
+default_typemap = {Type.Max : {Type.Max : 1, Type.Min : 0, Type.Avg : 0, Type.Var : 0},
+                   Type.Min : {Type.Max : 0, Type.Min : 1, Type.Avg : 0, Type.Var : 0},
                    Type.Avg : {Type.Max : 1, Type.Min : 1, Type.Avg : 1, Type.Var : 1}}
 
 def make_typemap():
diff --git a/parse/sched.py b/parse/sched.py
index 147a2e5..1213f0d 100644
--- a/parse/sched.py
+++ b/parse/sched.py
@@ -68,7 +68,7 @@ def make_iterator(fname):
     '''Iterate over (parsed record, processing method) in a
     sched-trace file.'''
     if not os.path.getsize(fname):
-        sys.stderr.write("Empty sched_trace file %s!" % fname)
+        # Likely a release master CPU
         return
 
     f = open(fname, 'rb')
@@ -176,6 +176,10 @@ def extract_sched_data(result, data_dir, work_dir):
 
     # Group per-task values
     for tdata in task_dict.itervalues():
+        if not tdata.params:
+            # Currently unknown where these invalid tasks come from...
+            continue
+
         miss_ratio = float(tdata.misses.num) / tdata.jobs
         # Scale average down to account for jobs with 0 tardiness
         avg_tard = tdata.misses.avg * miss_ratio
diff --git a/parse/tuple_table.py b/parse/tuple_table.py
index 491ea7b..47fb6b6 100644
--- a/parse/tuple_table.py
+++ b/parse/tuple_table.py
@@ -78,7 +78,7 @@ class ReducedTupleTable(TupleTable):
                 val = kv[col]
 
                 try:
-                    float(val)
+                    float(str(val))
                 except:
                     # Only vary numbers. Otherwise, just have seperate files
                     continue
@@ -93,9 +93,6 @@ class ReducedTupleTable(TupleTable):
         Leaf = namedtuple('Leaf', ['stat', 'variable', 'base',
                                    'summary', 'config', 'values'])
 
-        def next_type(path):
-            return path.pop() if path[-1] in Type else Type.Avg
-
         def leafs():
             for path, node in dir_map.leafs():
                 # The path will be of at least size 1: the filename
diff --git a/parse_exps.py b/parse_exps.py
index 8aa9b43..c254536 100755
--- a/parse_exps.py
+++ b/parse_exps.py
@@ -32,6 +32,9 @@ def parse_args():
     parser.add_option('-m', '--write-map', action='store_true', default=False,
                       dest='write_map',
                       help='Output map of values instead of csv tree')
+    parser.add_option('-p', '--processors', default=max(cpu_count() - 1, 1),
+                      type='int', dest='processors',
+                      help='number of threads for processing')
 
     return parser.parse_args()
 
@@ -134,7 +137,7 @@ def main():
 
     sys.stderr.write("Parsing data...\n")
 
-    procs = min(len(exps), max(cpu_count()/2, 1))
+    procs = min(len(exps), opts.processors)
     pool = Pool(processes=procs)
     pool_args = zip(exps, [opts.force]*len(exps))
     enum = pool.imap_unordered(parse_exp, pool_args, 1)
@@ -161,8 +164,8 @@ def main():
 
     reduced_table = result_table.reduce()
 
-    sys.stderr.write("Writing result...\n")
     if opts.write_map:
+        sys.stderr.write("Writing python map into %s...\n" % opts.out)
         # Write summarized results into map
         reduced_table.write_map(opts.out)
     else:
@@ -177,6 +180,7 @@ def main():
                     for e in exp:
                         print(e)
         else:
+            sys.stderr.write("Writing csvs into %s...\n" % opts.out)
             dir_map.write(opts.out)
 
 if __name__ == '__main__':
diff --git a/plot/style.py b/plot/style.py
index 21c4e7e..4e2057f 100644
--- a/plot/style.py
+++ b/plot/style.py
@@ -7,7 +7,8 @@ class Style(namedtuple('SS', ['marker', 'line', 'color'])):
 
 class StyleMap(object):
     '''Maps configs (dicts) to specific line styles.'''
-    DEFAULT = Style('', '-', 'k')
+    DEFAULT = Style(marker='', line= '-', color='k')
+    ORDER = [ str, bool, float, int ]
 
     def __init__(self, col_list, col_values):
         '''Assign (some) columns in @col_list to fields in @Style to vary, and
@@ -15,6 +16,17 @@ class StyleMap(object):
         self.value_map = {}
         self.field_map = {}
 
+        # Prioritize non-numbers
+        def type_priority(column):
+            value = col_values[column].pop()
+            col_values[column].add(value)
+            try:
+                t = float if float(value) % 1.0 else int
+            except:
+                t = bool if value in ['True','False'] else str
+            return StyleMap.ORDER.index(t)
+        col_list = sorted(col_list, key=type_priority)
+
         # TODO: undo this, switch to popping mechanism
         for field, values in reversed([x for x in self.__get_all()._asdict().iteritems()]):
             if not col_list:
@@ -31,9 +43,9 @@ class StyleMap(object):
 
     def __get_all(self):
         '''A Style holding all possible values for each property.'''
-        return Style(list('.,ov^<>1234sp*hH+xDd|_'), # markers
-                     ['-', ':', '--'], # lines
-                     list('bgrcmyk'))  # colors
+        return Style(marker=list('.,ov^<>1234sp*hH+xDd|_'),
+                     line=['-', ':', '--'],
+                     color=list('bgrcmyk'))
 
     def get_style(self, kv):
         '''Translate column values to unique line style.'''
diff --git a/plot_exps.py b/plot_exps.py
index 3b5636b..76e7396 100755
--- a/plot_exps.py
+++ b/plot_exps.py
@@ -7,11 +7,11 @@ import shutil as sh
 import sys
 import traceback
 from collections import namedtuple
+from multiprocessing import Pool, cpu_count
 from optparse import OptionParser
 from parse.col_map import ColMap,ColMapBuilder
 from parse.dir_map import DirMap
 from plot.style import StyleMap
-from multiprocessing import Pool, cpu_count
 
 def parse_args():
     parser = OptionParser("usage: %prog [options] [csv_dir]...")
@@ -20,6 +20,9 @@ def parse_args():
                       help='directory for plot output', default='plot-data')
     parser.add_option('-f', '--force', action='store_true', default=False,
                       dest='force', help='overwrite existing data')
+    parser.add_option('-p', '--processors', default=max(cpu_count() - 1, 1),
+                      type='int', dest='processors',
+                      help='number of threads for processing')
 
     return parser.parse_args()
 
@@ -97,7 +100,7 @@ def plot_wrapper(details):
     except:
         traceback.print_exc()
 
-def plot_dir(data_dir, out_dir, force):
+def plot_dir(data_dir, out_dir, max_procs, force):
     sys.stderr.write("Reading data...\n")
     dir_map = DirMap.read(data_dir)
 
@@ -119,7 +122,7 @@ def plot_dir(data_dir, out_dir, force):
     if not plot_details:
         return
 
-    procs = min(len(plot_details), max(cpu_count()/2, 1))
+    procs = min(len(plot_details), max_procs)
     pool  = Pool(processes=procs)
     enum  = pool.imap_unordered(plot_wrapper, plot_details)
 
@@ -150,7 +153,7 @@ def main():
             out_dir = "%s/%s" % (opts.out_dir, os.path.split(dir)[1])
         else:
             out_dir = opts.out_dir
-        plot_dir(dir, out_dir, opts.force)
+        plot_dir(dir, out_dir, opts.processors, opts.force)
 
 if __name__ == '__main__':
     main()