1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
|
import numpy
from util import load_binary_file
from stats import iqr_remove_outliers, iqr_cutoff
def get_data(fname, scale, extent, cutoff=None, maxval=1000.0,
stdev=False, manual=None):
data = load_binary_file(fname)
if cutoff and len(data) > cutoff:
data = data[:cutoff]
if not scale is None:
data *= scale
data.sort()
if stdev:
# standard deviations filter
std = numpy.std(data, ddof=1)
avg = numpy.mean(data)
lower = avg - std * extent
upper = avg + std * extent
elif extent:
# IQR filter
lower, upper = iqr_cutoff(data, extent)
else:
lower = 0
upper = maxval
min_idx, max_idx = numpy.searchsorted(data, [lower, upper])
if manual:
max_idx -= manual
return [data, max_idx, min_idx, upper, lower]
def compact_file(*args, **kargs):
data, max_idx, min_idx, iqr_max, iqr_min = get_data(*args, **kargs)
samples = data[min_idx:max_idx]
filtered = len(data) - len(samples)
max = samples[-1]
min = samples[0]
med = numpy.median(samples)
avg = numpy.mean(samples)
std = numpy.std(samples, ddof=1)
var = numpy.var(samples)
return [len(data), filtered, max, avg, min, med, std, var, iqr_max, iqr_min]
|
