Add numpy/scipy based statistics and helpers.

2 files changed, 40 insertions, 0 deletions

diff --git a/stats.py b/stats.py
new file mode 100644
index 0000000..f6c4401
--- /dev/null
+++ b/stats.py
@@ -0,0 +1,29 @@
+import scipy.stats as s
+import numpy as np
+
+from util import select
+
+
+def iqr(vect):
+    "return inter-quartile range of a vector"
+    q25 = s.scoreatpercentile(vect, 25)
+    q75 = s.scoreatpercentile(vect, 75)
+    return (q75 - q25, q25, q75)
+
+def cutoff_max(vect, percentile=99):
+    return s.scoreatpercentile(vect, percentile)
+
+def iqr_is_not_outlier(table, col=1, extend=1.5):
+    "create a filter function that flags outliers"
+    (spread, low, high) = iqr(table[:,col])
+    min_val = low  - extend * spread
+    max_val = high + extend * spread
+    return lambda row: min_val <= row[col] <= max_val
+
+def iqr_remove_outliers(table, col=1, extend=1.5):
+    """Return a copy that only includes rows that
+    fall within the IQR-based window.
+    """
+    valid = iqr_is_not_outlier(table, col, extend)
+    return select(valid, table)
+
diff --git a/util.py b/util.py
new file mode 100644
index 0000000..fdedaa4
--- /dev/null
+++ b/util.py
@@ -0,0 +1,11 @@
+import numpy as np
+
+def load_csv_file(fname, *args, **kargs):
+    return np.genfromtxt(open(fname), delimiter=",", comments="#",
+                         *args, **kargs)
+
+def select(keep, rows):
+    ok_rows = np.zeros(len(rows), dtype=bool)
+    for i, row in enumerate(rows):
+        ok_rows[i] = keep(row)
+    return rows[ok_rows]