3 files changed, 198 insertions, 21 deletions

diff --git a/pm_data_analysis/pm_data_analyzer.py b/pm_data_analysis/pm_data_analyzer.py
new file mode 100755
index 0000000..6862f98
--- /dev/null
+++ b/pm_data_analysis/pm_data_analyzer.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+#
+# Preemption and migration overheads analysis.
+#    Take a single-task-set-size file and generate max / avg for the valid
+#    samples for preemption and different kinds of migration
+#
+#    Save computed valid overheads and print a file suitable for processing
+#    using gnuplot / tikz / lua
+
+import sys
+import numpy as np
+
+# preemption and migration C data exchanger
+import pm
+import pmserialize as pms
+
+from optparse import OptionParser
+
+class Overhead:
+    def __init__(self):
+        self.overheads = []
+        self.index = 0
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        if self.index == len(self.overheads):
+            self.index = 0
+            raise StopIteration
+        self.index += 1
+        return self.overheads[self.index - 1]
+
+    def add(self, ovd_vector, label):
+        self.overheads.append([ovd_vector, label])
+
+
+class InterQuartileRange:
+    def __init__(self, low, high):
+        self.low = low
+        self.high = high
+
+    def remOutliers(self, vector):
+        # discard points etc
+        return vector
+
+def read_valid_data(filename, coresL2, valid_ovds):
+    suff = filename.find('.raw')
+    if suff == -1:
+        print "Warning: bad suffix, I may fail in finding the right files"
+
+    nf = filename[0:suff] + '_preemption.vbin'
+    valid_ovds.add(pms.unpickl_it(nf), 'preemtion')
+    nf = filename[0:suff] + '_onchip.vbin'
+    valid_ovds.add(pms.unpickl_it(nf), 'onchip')
+    nf = filename[0:suff] + '_offchip.vbin'
+    valid_ovds.add(pms.unpickl_it(nf), 'offchip')
+    if coresL2 != 0:
+        nf = filename[0:suff] + '_l2cache.vbin'
+        valid_ovds.add(pms.unpickl_it(nf), 'l2cache')
+
+
+def process_raw_data(filename, coresL2, coresC, valid_ovds):
+    # initialize pmmodule
+    pm.load(filename, coresL2, coresC)
+
+    ovds = Overhead()
+
+    # get overheads
+    ovds.add(pm.getPreemption(), 'preemption')
+    ovds.add(pm.getOnChipMigration(), 'onchip')
+    ovds.add(pm.getOffChipMigration(), 'offchip')
+    if coresL2 != 0:
+        ovds.add(pm.getL2Migration(), 'l2cache')
+
+    if verbose:
+        for i in ovds:
+            print i[0], i[1]
+
+    # instance the statistical analizer to remove outliers
+    sd = InterQuartileRange(25,75)
+
+    for i in ovds:
+        valid_ovds.add(sd.remOutliers(i[0]), i[1])
+
+    # serialize valid overheads
+    for i in valid_ovds:
+        suff = filename.find('.raw')
+        if suff == -1:
+            print "Warning: ugly name for outfiles"
+
+        curf = filename[0:suff] + '_' + i[1] + '.vbin'
+        pms.pickl_it(i[0], curf)
+
+    del ovds
+
+# tssize_char is the number of chars used to represent the tssize
+# e.g., 050 -> tsize_char = 3
+def analize_data(valid_ovds, filename, tssize_char):
+    suff = filename.find('.raw')
+    if suff == -1:
+        print filename
+        print "Warning: ugly name for outfiles; check number of tasks!"
+
+    nf = filename[0:suff - tssize_char - 1] + '.csv'
+    csvf = open(nf, 'a')
+
+    # taskset size
+    pms.cvs_it(csvf, filename[suff - tssize_char:suff])
+
+    for i in valid_ovds:
+        # overhead type
+        pms.cvs_it(csvf, i[1])
+        # data (atm just overhead, not length)
+        vector = i[0][:,0]
+        if vector != []:
+            pms.cvs_it(csvf, "%5.5f" % np.max(vector))
+            pms.cvs_it(csvf, "%5.5f" % np.average(vector))
+        else:
+            pms.cvs_it(csvf, "[]")
+            pms.cvs_it(csvf, "[]")
+
+    csvf.close()
+
+# filename-extension convention to get "pretty" output filenames
+# .raw for raw bin data
+# .vbin for valid overheads
+# .csv for processed final data
+def main():
+    usage = "Usage: %prog [options] filename"
+    description = "FILENAME is where the .raw overhead data are. Filename \
+            and the path to it also gives the base path and filename for the \
+            files that contains already processed overheads and the directory \
+            where to save the output data."
+    parser = OptionParser(usage=usage, description=description)
+    parser.add_option("-l", "--cores-per-l2", dest="coresL2",
+                      action="store", type="int", default="0",
+            help="number of cores per L2 cache;  "
+                 "if all cores share the same L2 (i.e., no L3) set this to 0 "
+                 "(default = 0)")
+    parser.add_option("-c", "--cores-per-chip", dest="coresC",
+                      action="store", type="int", default="4",
+            help="number of cores per chip (default = 4)")
+    parser.add_option("-r", "--read-valid-data", dest="read_valid",
+                      action="store_true", default=False,
+                      help="read already processed data from file")
+    parser.add_option("-v", "--verbose", dest="verbose",
+                      action="store_true", default=False,
+                      help="Be verbose")
+    (options, args) = parser.parse_args()
+    if len(args) != 1:
+        parser.error("Argument missing")
+        sys.exit(-1)
+
+    valid_ovds = Overhead()
+
+    global verbose
+    if options.verbose:
+        verbose = 1
+    else:
+        verbose = 0
+
+    if options.read_valid:
+        read_valid_data(args[0], options.coresL2, valid_ovds)
+    else:
+        process_raw_data(args[0], options.coresL2, options.coresC,
+            valid_ovds)
+
+    analize_data(valid_ovds, args[0], 3)
+
+if __name__ == '__main__':
+    main()
+
diff --git a/pm_data_analysis/pmserialize.py b/pm_data_analysis/pmserialize.py
new file mode 100644
index 0000000..14e0f3b
--- /dev/null
+++ b/pm_data_analysis/pmserialize.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import cPickle
+import csv
+
+# http://en.wikipedia.org/wiki/Picolit
+def pickl_it(vector, filename):
+    f = open(filename, 'w')
+    cPickle.dump(vector, f)
+    f.close()
+
+def unpickl_it(filename):
+    f = open(filename, 'r')
+    vector = cPickle.load(f)
+    f.close()
+    return vector
+
+# FIXME the one char one comma format from csv
+# is not very "human readable" :|
+def cvs_it(file, string):
+    wr = csv.writer(file)
+    wr.writerow(string)
+
+def uncvs_it(file, string):
+    return None
diff --git a/test_get_arrays.py b/test_get_arrays.py
deleted file mode 100644
index 74af08e..0000000
--- a/test_get_arrays.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/python
-import sys, pm
-
-args = sys.argv[1:]
-if len(args) != 1:
-    print "Filename required"
-    sys.exit(-1)
-
-pm.load(args[0],0,4)
-x = pm.getPreemption()
-y = pm.getOnChipMigration()
-z = pm.getL2Migration()
-w = pm.getOffChipMigration()
-print "preemption: "
-print x
-print "samechip:"
-print y
-print "l2:"
-print z
-print "offchip"
-print w