Add tools for overhead-processing workflow

7 files changed, 595 insertions, 0 deletions

diff --git a/ft-combine-samples b/ft-combine-samples
new file mode 100755
index 0000000..a1f19ed
--- /dev/null
+++ b/ft-combine-samples
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+STRIP_CMD=""
+
+function add_strip()
+{
+    TAG=$1
+    STRIP_CMD="$STRIP_CMD -e s/_${TAG}=[^_]*//"
+}
+
+while true
+do
+    case "$1" in
+        -n | --task-count)
+            shift
+            add_strip n
+            ;;
+
+        -c | --cpu)
+            shift
+            add_strip cpu
+            ;;
+
+        -m | --msg)
+            shift
+            add_strip msg
+            ;;
+
+        -s | --seq)
+            shift
+            add_strip seq
+            ;;
+
+        -u | --util)
+            shift
+            add_strip u
+            ;;
+
+        -l | --locks)
+            shift
+            add_strip locks
+            ;;
+
+        -x | --custom)
+            shift
+            add_strip $1
+            shift
+            ;;
+
+        --std)
+            shift
+            add_strip n
+            add_strip cpu
+            add_strip msg
+            add_strip seq
+            add_strip u
+            ;;
+
+      *)
+          break
+          ;;
+    esac
+done
+
+if [ -z "$STRIP_CMD" ]
+then
+    echo "Error: no fields to strip specified."
+    exit 1
+fi
+
+function do_append() {
+    TARGET=`basename $1 | sed $STRIP_CMD`
+    TARGET="combined-$TARGET"
+    printf "\n[$NUM/$TOTAL] Combining $1 -> $TARGET\n"
+    cat $1 >> $TARGET
+}
+
+TOTAL=$#
+NUM=0
+
+echo "File names will be mangled with: sed $STRIP_CMD"
+
+
+while [ "" != "$*" ]
+do
+    NUM=$((NUM + 1))
+    do_append "$1"
+    shift
+done
diff --git a/ft-compute-stats b/ft-compute-stats
new file mode 100755
index 0000000..e07fd16
--- /dev/null
+++ b/ft-compute-stats
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+
+from __future__ import division
+
+import numpy
+
+import optparse
+import sys
+import os
+
+from os.path import splitext
+
+
+def decode_key_value_filename(name):
+    "Map key=value_otherkey=other-value names to proper dictionary."
+    params = {}
+    parts = name.split('_')
+    for p in parts:
+        kv = p.split('=')
+        k = kv[0]
+        v = kv[1] if len(kv) > 1 else None
+        params[k] = v
+    return params
+
+
+def stats_for_file(fname, scale):
+    n    = 0
+    max  = 0
+    p95  = 0
+    p99  = 0
+    p999 = 0
+    min  = 0
+    med  = 0
+    avg  = 0
+    std  = 0
+    var  = 0
+
+    size = os.stat(fname).st_size
+    if size:
+        samples = numpy.memmap(fname, dtype='float32', mode='c')
+
+        n = len(samples)
+        if n > 0:
+            samples *= scale
+            max  = numpy.amax(samples)
+            p95  = numpy.percentile(samples, 95.0)
+            p99  = numpy.percentile(samples, 99.0)
+            p999 = numpy.percentile(samples, 99.9)
+            med  = numpy.median(samples)
+            avg  = numpy.mean(samples)
+            min  = numpy.amin(samples)
+
+            std = numpy.std(samples, ddof=1)
+            var = numpy.var(samples)
+
+    return [n, max, p999, p99, p95, avg, med, min, std, var]
+
+o = optparse.make_option
+
+opts = [
+    o('-p', '--cycles-per-usec', action='store', dest='cycles', type='float',
+      help='how many cycles per usec'),
+    ]
+
+defaults = {
+    'cycles'  : None,
+    }
+
+options = None
+
+def fmt_cell(x):
+    if type(x) == str:
+        return "%25s" % x
+    if type(x) == int:
+        return "%25d" % x
+    else:
+        return "%25.5f" % x
+
+def write_header():
+    labels = ["Plugin", "#cores", "Overhead", 'Unit', "#tasks",
+              "#samples",
+              "max", "99.9th perc.", "99th perc.", "95th perc.",
+              "avg", "med", "min", "std", "var", "file"]
+    header = ", ".join(fmt_cell(x) for x in labels)
+    print '#%s' % header[1:]
+
+
+def stats_file(fname):
+    name, ext = splitext(fname)
+    conf = decode_key_value_filename(name)
+
+    if 'overhead' in conf and conf['overhead'].rfind('-LATENCY') != -1:
+        # latency is stored in nanoseconds, not cycles
+        scale = 1 / 1000 # convert from nanoseconds
+        unit = 'microseconds (scale = 1/1000)'
+    elif options.cycles is None:
+        scale = 1
+        unit = 'cycles'
+    else:
+        # convert from cycles to usec
+        scale = 1 / options.cycles
+        unit = 'microseconds (scale = 1/%f)' % options.cycles
+
+    stats = stats_for_file(fname, scale)
+    if 'locks' in conf:
+        sched = '%s_locks=%s' % (conf['scheduler'], conf['locks'])
+    elif 'scheduler' in conf:
+        sched = conf['scheduler']
+    else:
+        sched = 'UNKNOWN'
+
+    ohead = conf['overhead'] if 'overhead' in conf else 'UNKNOWN'
+    n = conf['n'] if 'n' in conf else '*'
+    m = conf['m'] if 'm' in conf else '*'
+
+    info = [sched, m, ohead, unit, n]
+    finfo = [fname]
+    print ", ".join([fmt_cell(x) for x in  info + stats + finfo])
+    sys.stdout.flush()
+
+if __name__ == '__main__':
+    # FIXME: would be nicer with argparse
+    parser = optparse.OptionParser(option_list=opts)
+    parser.set_defaults(**defaults)
+    (options, files) = parser.parse_args()
+
+    try:
+        write_header()
+        for f in files:
+            try:
+                stats_file(f)
+            except IOError, msg:
+                print >> sys.stderr, msg
+    except KeyboardInterrupt:
+        pass
diff --git a/ft-count-samples b/ft-count-samples
new file mode 100755
index 0000000..b08f5fd
--- /dev/null
+++ b/ft-count-samples
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+EVENTS=""
+for F in $*
+do
+    E=`echo $F | sed -e 's/.*overhead=\([^_.]*\).*/\1/'`
+    EVENTS="$EVENTS $E"
+done
+
+declare -A MATCHES
+
+for E in $EVENTS
+do
+    if [ -z "${MATCHES[$E]}" ]
+    then
+        MATCHES[$E]=`ls $* | egrep "_overhead=${E}[_.]"`
+    fi
+done
+
+PATH_TO_SCRIPT=`dirname $0`
+function find_helper()
+{
+        IN_PATH=`which $1`
+        if [ -z "$IN_PATH" ] && [ ! -z "$PATH_TO_SCRIPT" ] &&  [ -x "$PATH_TO_SCRIPT/$1" ]
+        then
+                echo "$PATH_TO_SCRIPT/$1"
+        else
+                echo "$IN_PATH"
+        fi
+}
+
+function die()
+{
+        echo "Error: $*"
+        exit 1
+}
+
+SHUFFLE_TRUNCATE=`find_helper shuffle_truncate.py`
+[ -z "$SHUFFLE_TRUNCATE" ] && die "Can't find 'shuffle_truncate.py' utility."
+
+for E in ${!MATCHES[@]}
+do
+    FILES="${MATCHES[$E]}"
+    if [ ! -z "$FILES" ]
+    then
+        COUNT=`$SHUFFLE_TRUNCATE --count --only-min $FILES`
+            printf "%20s, %7d\n" "$E" "$COUNT"
+    fi
+done
+
diff --git a/ft-extract-samples b/ft-extract-samples
new file mode 100755
index 0000000..fd8a89c
--- /dev/null
+++ b/ft-extract-samples
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+PATH_TO_SCRIPT=`dirname $0`
+function find_helper()
+{
+        IN_PATH=`which $1`
+        REL_TO_PATH="$PATH_TO_SCRIPT/$2"
+        if [ -z "$IN_PATH" ] && [ ! -z "$PATH_TO_SCRIPT" ] &&  [ -x "$PATH_TO_SCRIPT/$1" ]
+        then
+                echo "$PATH_TO_SCRIPT/$1"
+        elif [ -z "$IN_PATH" ] && [ ! -z "$PATH_TO_SCRIPT" ] &&  [ -x "$REL_TO_PATH/$1" ]
+        then
+            echo "$REL_TO_PATH/$1"
+        else
+                echo "$IN_PATH"
+        fi
+}
+
+function die()
+{
+        echo "Error: $*"
+        exit 1
+}
+
+SPLITTER=`find_helper ft2csv ../feather-trace-tools`
+[ -z "$SPLITTER" ] && die "Can't find 'ftsort' utility."
+
+
+BE_EVENTS="SEND_RESCHED SEND_XCALL QUANTUM_BOUNDARY"
+declare -A XTRA_OPTS
+for E in $BE_EVENTS
+do
+    XTRA_OPTS[$E]="-b"
+done
+
+# for future extension...
+IRQ_EVENTS=""
+for E in $IRQ_EVENTS
+do
+    XTRA_OPTS[$E]="-x ${XTRA_OPTS[$E]}"
+done
+
+OPTS="-r"
+OLD_EXT=bin
+EXT=float32 # NumPy float32 dtype format
+
+function do_split() {
+        printf "\n[$NUM/$TOTAL] Extracting samples from $1\n"
+        PRESENT=`$SPLITTER -l "$1" | sed -e 's/_START//' -e 's/_END//' | sort | uniq`
+        echo $1 '->' $PRESENT
+
+        for E in $PRESENT; do
+            EP=${E/_/-}
+            WHERE=`basename "$1" | sed -e "s/[.]${OLD_EXT}//"`
+            TARGET="${WHERE}_overhead=$EP.$EXT"
+            echo $1 $E ">>" $TARGET
+            $SPLITTER $OPTS ${XTRA_OPTS[$E]} $E "$1" >> $TARGET
+        done
+}
+
+if [ ! -f "$1" ]; then
+    echo  "Usage: ft-extract-samples <FEATHER-TRACE-FILE.bin>+"
+    exit 1
+fi
+
+TOTAL=$#
+NUM=0
+
+
+while [ "" != "$*" ]
+do
+    NUM=$((NUM + 1))
+    do_split "$1"
+    shift
+done
diff --git a/ft-select-samples b/ft-select-samples
new file mode 100755
index 0000000..fb3c39c
--- /dev/null
+++ b/ft-select-samples
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -e
+
+COUNTS="$1"
+shift
+
+EVENTS=""
+for F in $*
+do
+    E=`echo $F | sed -e 's/.*overhead=\([^_.]*\).*/\1/'`
+    EVENTS="$EVENTS $E"
+done
+
+declare -A MATCHES
+
+for E in $EVENTS
+do
+    if [ -z "${MATCHES[$E]}" ]
+    then
+        MATCHES[$E]=`ls $* | egrep "_overhead=${E}[_.]"`
+    fi
+done
+
+PATH_TO_SCRIPT=`dirname $0`
+function find_helper()
+{
+        IN_PATH=`which $1`
+        if [ -z "$IN_PATH" ] && [ ! -z "$PATH_TO_SCRIPT" ] &&  [ -x "$PATH_TO_SCRIPT/$1" ]
+        then
+                echo "$PATH_TO_SCRIPT/$1"
+        else
+                echo "$IN_PATH"
+        fi
+}
+
+function die()
+{
+        echo "Error: $*"
+        exit 1
+}
+
+SHUFFLE_TRUNCATE=`find_helper ft-shuffle-truncate`
+[ -z "$SHUFFLE_TRUNCATE" ] && die "Can't find 'ft-shuffle-truncate' utility."
+
+EVENTS=`awk -F ',' '{print $1}' $COUNTS`
+
+for E in ${!MATCHES[@]}
+do
+    FILES="${MATCHES[$E]}"
+    CUTOFF=`awk -F ',' "/ $E,/ {print \\$2}" $COUNTS`
+    if [ -z "$FILES" ]
+    then
+        echo "[EE] No files for $E."
+    elif (( $CUTOFF == 0 ))
+    then
+        echo "[EE] Cutoff for $E is zero."
+    else
+        echo "$E -> $CUTOFF"
+        $SHUFFLE_TRUNCATE -c $CUTOFF $FILES
+    fi
+done
diff --git a/ft-shuffle-truncate b/ft-shuffle-truncate
new file mode 100755
index 0000000..44ace39
--- /dev/null
+++ b/ft-shuffle-truncate
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+
+import numpy
+import os
+import sys
+import optparse
+
+def load_binary_file(fname, dtype='float32', modify=False):
+    size = os.stat(fname).st_size
+
+    if size:
+        data = numpy.memmap(fname, dtype=dtype,
+                            mode='r+' if modify else 'c')
+        return data
+    else:
+        return []
+
+o = optparse.make_option
+
+opts = [
+    o('-c', '--cut-off', action='store', dest='cutoff', type='int',
+      help='max number of samples to use'),
+
+    o(None, '--count', action='store_true', dest='count',
+      help='just report the number of samples in each file'),
+
+    o(None, '--only-min', action='store_true', dest='only_min',
+      help='When counting, report only the minimum number of samples.'),
+
+    o(None, '--output-dir', action='store', dest='output_dir',
+      help='directory where output files should be stored.')
+    ]
+
+defaults = {
+    'cutoff'  : None,
+    'count'   : False,
+    'only_min' : False,
+    'output_dir' : None,
+    }
+
+options = None
+
+def load_files(fnames):
+    return [load_binary_file(f) for f in fnames]
+
+def shuffle_truncate(arrays, fnames, target_length=None):
+    # Determine how many samples we can use.
+    if target_length:
+        shortest = target_length
+    else:
+        shortest = min([len(a) for a in arrays])
+        print "Selecting %d samples from each data file." % shortest
+
+    # Make sure we'll select samples from all
+    # parts of the data file.
+    for a, n in zip(arrays, fnames):
+        if len(a) > shortest:
+            # Gotta be uniformly shuffled.
+            print "Shuffling %s ..." % n
+            numpy.random.shuffle(a)
+        else:
+            # not enough samples
+            print "Not shuffling %s." % n
+
+    # Now select the same number of samples from each file.
+    truncated = [a[:shortest] for a in arrays]
+
+    return truncated
+
+def store_files(arrays, fnames):
+    for a, fn in zip(arrays, fnames):
+        print 'Storing %s.' % fn
+        fd = open(fn, 'wb')
+        a.tofile(fd)
+        fd.close()
+
+def target_file(fname, want_ext):
+    f = os.path.basename(fname)
+    if options.output_dir:
+        d = options.output_dir
+    else:
+        d = os.path.dirname(fname)
+    if not want_ext is None:
+        name, ext = os.path.splitext(f)
+        f = "%s.%s" % (name, want_ext)
+    return os.path.join(d, f)
+
+def shuffle_truncate_store(files, cutoff=None, ext='sf32'):
+    data  = load_files(files)
+    trunc = shuffle_truncate(data, files, target_length=cutoff)
+    names = [target_file(f, ext) for f in files]
+    store_files(trunc, names)
+
+def shuffle_truncate_store_individually(files, cutoff):
+    fmt = "%%0%dd" % len(str(len(files)))
+    for i, f in enumerate(files):
+        print ("["  + fmt + "/%d] %s") % (i+1, len(files),
+                                          os.path.basename(f))
+        sys.stdout.flush()
+        name = target_file(f, 'sbn')
+        fs = os.stat(f)
+        if os.path.exists(name):
+            print "Skipping since %s exists." % name
+        elif fs.st_size == 0:
+            print "Skipping since trace is empty."
+        else:
+            shuffle_truncate_store([f], cutoff=cutoff)
+
+def report_sample_counts(files):
+    counts = []
+    fmt = "%%0%dd" % len(str(len(files)))
+    for i, f in enumerate(files):
+        d = load_binary_file(f)
+        counts.append(len(d))
+        if not options.only_min:
+            print ("["  + fmt + "/%d] %8d %s") % (i+1, len(files), len(d), f)
+            sys.stdout.flush()
+        del d
+    if options.only_min:
+        print min(counts)
+
+if __name__ == '__main__':
+    # FIXME: would be nicer with argparse
+    parser = optparse.OptionParser(option_list=opts)
+    parser.set_defaults(**defaults)
+    (options, files) = parser.parse_args()
+
+    if not files:
+        print "Usage: ft-shuffle-truncate data1.float32 data2.float32 data3.float32 ..."
+    else:
+        if options.count:
+            report_sample_counts(files)
+        elif options.cutoff:
+            shuffle_truncate_store_individually(files, options.cutoff)
+        else:
+            shuffle_truncate_store(files)
diff --git a/ft-sort-traces b/ft-sort-traces
new file mode 100755
index 0000000..524737d
--- /dev/null
+++ b/ft-sort-traces
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+
+PATH_TO_SCRIPT=`dirname $0`
+function find_helper()
+{
+        IN_PATH=`which $1`
+        REL_TO_PATH="$PATH_TO_SCRIPT/$2"
+        if [ -z "$IN_PATH" ] && [ ! -z "$PATH_TO_SCRIPT" ] &&  [ -x "$PATH_TO_SCRIPT/$1" ]
+        then
+                echo "$PATH_TO_SCRIPT/$1"
+        elif [ -z "$IN_PATH" ] && [ ! -z "$PATH_TO_SCRIPT" ] &&  [ -x "$REL_TO_PATH/$1" ]
+        then
+            echo "$REL_TO_PATH/$1"
+        else
+                echo "$IN_PATH"
+        fi
+}
+
+function die()
+{
+        echo "Error: $*"
+        exit 1
+}
+
+SORT=`find_helper ftsort ../feather-trace-tools`
+[ -z "$SORT" ] && die "Can't find 'ftsort' utility."
+
+function do_sort() {
+        printf "[$NUM/$TOTAL] Sorting $1\n"
+        $SORT $1 2>&1
+}
+
+if [ ! -f "$1" ]; then
+    echo  "Usage: ft-sort-traces <FEATHER-TRACE-FILE.bin>+"
+    exit 1
+fi
+
+TOTAL=`echo $* | wc -w`
+
+NUM=0
+
+while [ "" != "$*" ]; do
+    NUM=$((NUM + 1))
+    do_sort $1
+    shift
+done