add scripts for shuffling and truncating sample files

3 files changed, 137 insertions, 0 deletions

diff --git a/count-all b/count-all
new file mode 100755
index 0000000..7a353bc
--- /dev/null
+++ b/count-all
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+EVENTS="SCHED SCHED2 TICK CXS RELEASE RELEASE-LATENCY SEND-RESCHED"
+
+DIR="$1"
+
+if [ -z "$DIR" ]
+then
+    DIR=.
+fi
+
+for E in $EVENTS
+do
+    FILES=`find "$DIR" -iname "*overhead=${E}.bin"`
+    shuffle_truncate.py --count $FILES > counts_overhead=${E}.txt
+done
+
diff --git a/shuffle-truncate-all b/shuffle-truncate-all
new file mode 100755
index 0000000..b83c360
--- /dev/null
+++ b/shuffle-truncate-all
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+COUNTS="$1"
+DIR="$2"
+
+if [ -z "$DIR" ]
+then
+    DIR=.
+fi
+
+EVENTS=`awk '{print $1}' $COUNTS`
+
+for E in $EVENTS
+do
+    CUTOFF=`grep $E $COUNTS | awk '{print $2}'`
+    FILES=`find "$DIR" -iname "*overhead=${E}.bin"`
+    shuffle_truncate.py -c $CUTOFF $FILES
+done
diff --git a/shuffle_truncate.py b/shuffle_truncate.py
new file mode 100755
index 0000000..6a48ca2
--- /dev/null
+++ b/shuffle_truncate.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+
+import numpy
+import os
+import sys
+import optparse
+
+from util import load_binary_file
+
+o = optparse.make_option
+
+opts = [
+    o('-c', '--cut-off', action='store', dest='cutoff', type='int',
+      help='max number of samples to use'),
+
+    o(None, '--count', action='store_true', dest='count',
+      help='just report the number of samples in each file'),
+
+    ]
+
+defaults = {
+    'cutoff'  : None,
+    'count'   : False,
+    }
+
+options = None
+
+def load_files(fnames):
+    return [load_binary_file(f) for f in fnames]
+
+def shuffle_truncate(arrays, fnames, target_length=None):
+    # Determine how many samples we can use.
+    if target_length:
+        shortest = target_length
+    else:
+        shortest = min([len(a) for a in arrays])
+        print "Selecting %d samples from each data file." % shortest
+
+    # Make sure we'll select samples from all
+    # parts of the data file.
+    for a, n in zip(arrays, fnames):
+        if len(a) > shortest:
+            # Gotta be uniformly shuffled.
+            print "Shuffling %s ..." % n
+            numpy.random.shuffle(a)
+
+    # Now select the same number of samples from each file.
+    truncated = [a[:shortest] for a in arrays]
+
+    return truncated
+
+def store_files(arrays, fnames):
+    for a, fn in zip(arrays, fnames):
+        print 'Storing %s.' % fn
+        fd = open(fn, 'wb')
+        a.tofile(fd)
+        fd.close()
+
+def target_file(fname, want_ext):
+    f = os.path.basename(fname)
+    if not want_ext is None:
+        name, ext = os.path.splitext(f)
+        return "%s.%s" % (name, want_ext)
+    else:
+        return f
+
+def shuffle_truncate_store(files, cutoff=None, ext='sbn'):
+    data  = load_files(files)
+    trunc = shuffle_truncate(data, files, target_length=cutoff)
+    names = [target_file(f, ext) for f in files]
+    store_files(trunc, names)
+
+def shuffle_truncate_store_individually(files, cutoff):
+    fmt = "%%0%dd" % len(str(len(files)))
+    for i, f in enumerate(files):
+        print ("["  + fmt + "/%d] %s") % (i+1, len(files),
+                                          os.path.basename(f))
+        sys.stdout.flush()
+        shuffle_truncate_store([f], cutoff=cutoff)
+
+def report_sample_counts(files):
+    fmt = "%%0%dd" % len(str(len(files)))
+    for i, f in enumerate(files):
+        d = load_binary_file(f)
+        print ("["  + fmt + "/%d] %8d %s") % (i+1, len(files), len(d), f)
+        sys.stdout.flush()
+        del d
+
+if __name__ == '__main__':
+    parser = optparse.OptionParser(option_list=opts)
+    parser.set_defaults(**defaults)
+    (options, files) = parser.parse_args()
+
+    if not files:
+        print "Usage: shuffle_truncate_py data1.bin data2.bin data3.bin ..."
+    else:
+        if options.count:
+            report_sample_counts(files)
+        elif options.cutoff:
+            shuffle_truncate_store_individually(files, options.cutoff)
+        else:
+            shuffle_truncate_store(files)