2 files changed, 71 insertions, 14 deletions

diff --git a/pm_data_analysis/pm_data_analyzer.py b/pm_data_analysis/pm_data_analyzer.py
index 6862f98..644a9aa 100755
--- a/pm_data_analysis/pm_data_analyzer.py
+++ b/pm_data_analysis/pm_data_analyzer.py
@@ -13,6 +13,7 @@ import numpy as np
 # preemption and migration C data exchanger
 import pm
 import pmserialize as pms
+import statanalyzer as pmstat
 
 from optparse import OptionParser
 
@@ -34,16 +35,6 @@ class Overhead:
     def add(self, ovd_vector, label):
         self.overheads.append([ovd_vector, label])
 
-
-class InterQuartileRange:
-    def __init__(self, low, high):
-        self.low = low
-        self.high = high
-
-    def remOutliers(self, vector):
-        # discard points etc
-        return vector
-
 def read_valid_data(filename, coresL2, valid_ovds):
     suff = filename.find('.raw')
     if suff == -1:
@@ -78,10 +69,19 @@ def process_raw_data(filename, coresL2, coresC, valid_ovds):
             print i[0], i[1]
 
     # instance the statistical analizer to remove outliers
-    sd = InterQuartileRange(25,75)
+    sd = pmstat.InterQuartileRange(25,75, True)
 
     for i in ovds:
-        valid_ovds.add(sd.remOutliers(i[0]), i[1])
+        # just add overheads, "forget" preemption length
+        valid_ovds.add(sd.remOutliers(i[0][:,0]), i[1])
+
+    if verbose:
+        # check outliers removals
+        for i in ovds:
+            print i[1], len(i[0])
+        print "\nAfter outliers removal:"
+        for i in valid_ovds:
+            print i[1], len(i[0])
 
     # serialize valid overheads
     for i in valid_ovds:
@@ -111,8 +111,9 @@ def analize_data(valid_ovds, filename, tssize_char):
     for i in valid_ovds:
         # overhead type
         pms.cvs_it(csvf, i[1])
-        # data (atm just overhead, not length)
-        vector = i[0][:,0]
+        # data (valid_ovds already have only overheads, not length)
+        # vector = i[0][:,0]
+        vector = i[0]
         if vector != []:
             pms.cvs_it(csvf, "%5.5f" % np.max(vector))
             pms.cvs_it(csvf, "%5.5f" % np.average(vector))
diff --git a/pm_data_analysis/statanalyzer.py b/pm_data_analysis/statanalyzer.py
new file mode 100644
index 0000000..6e096c4
--- /dev/null
+++ b/pm_data_analysis/statanalyzer.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import numpy as np
+from scipy import stats
+
+class InterQuartileRange:
+    def __init__(self, low, high, extend = False):
+        self.low = low
+        self.high = high
+        # extend is 1.5 extension of IQR
+        self.extend = extend
+
+    def remOutliers(self, vector):
+        svect = np.sort(vector)
+        q1 = stats.scoreatpercentile(svect, self.low)
+        q3 = stats.scoreatpercentile(svect, self.high)
+
+        # match the values \in svect which are closer to q[1|3]
+        # (q1, q3)
+        q1_pos = -1
+        q3_pos = -1
+        cur_pos = 0
+        for i in svect:
+            if q1_pos != -1 and q3_pos != -1:
+                break
+            if q1_pos == -1 and i > q1:
+                q1_pos = cur_pos
+            if q3_pos == -1 and q3 < i:
+                q3_pos = cur_pos
+
+            cur_pos += 1
+
+        if self.extend == True:
+            # 1.5 IQR outliers elimination
+            eiqr = (svect[q3_pos] - svect[q1_pos]) * 1.5
+            eq1 = svect[q1_pos] - eiqr
+            if eq1 < svect[0]:
+                eq1 = svect[0]
+            eq3 = svect[q3_pos] + eiqr
+            if eq3 > svect[len(svect) - 1]:
+                eq3 = svect[len(svect) - 1]
+            # match the values \in svect which are closer to eq[1|3]
+            q1_pos = -1
+            q3_pos = -1
+            cur_pos = 0
+            for i in svect:
+                if q1_pos != -1 and q3_pos != -1:
+                    break
+                if q1_pos == -1 and i > eq1:
+                    q1_pos = cur_pos
+                if q3_pos == -1 and eq3 < i:
+                    q3_pos = cur_pos
+
+                cur_pos += 1
+
+        return svect[q1_pos : q3_pos]