CPMD and producer/consumer overhead scripts.

6 files changed, 639 insertions, 0 deletions

diff --git a/cache_cost.py b/cache_cost.py
new file mode 100755
index 0000000..9f4e54a
--- /dev/null
+++ b/cache_cost.py
@@ -0,0 +1,181 @@
+#!/usr/bin/python
+
+import os
+import copy
+import sys
+import string
+import smtplib
+import socket
+import time
+import itertools
+import multiprocessing
+
+from run.executable.executable import Executable
+
+test_mode = False
+email_notification = True
+
+def run_exp(nsamples, ncpu, numa_args, wss, wcycle, sleep_range_ms, walk, do_pollute, do_huge_pages, do_uncache_pages):
+
+        fstub = 'pmo'
+        fstub += '_host='+socket.gethostname().split('.')[0]
+        fstub += '_ncpu='+str(ncpu)
+        fstub += '_wss='+str(wss)
+        fstub += '_wcycle='+str(wcycle)
+        fstub += '_smin='+str(sleep_range_ms[0]*1000)
+        fstub += '_smax='+str(sleep_range_ms[1]*1000)
+        fstub += '_polluters='+str(do_pollute)
+        fstub += '_hpages='+str(do_huge_pages)
+        fstub += '_upages='+str(do_uncache_pages)
+        fstub += '_walk='+walk
+
+        filename = fstub + '.csv'
+
+        if os.path.exists(filename):
+                return
+
+        polluters = []
+        if do_pollute:
+                for i in range(multiprocessing.cpu_count()):
+                        if numa_args:
+                                # number of CPUs in one of two NUMA nodes
+                                if ncpu == 6:
+                                        if i < 6:
+                                                numa_hack = ['--cpunodebind=0', '--interleave=0']
+                                        else:
+                                                numa_hack = ['--cpunodebind=1', '--interleave=1']
+                                else:
+                                        numa_hack = ['--cpunodebind=0,1', '--interleave=0,1']
+                                args = copy.deepcopy(numa_hack)
+                                args.append('memthrash')
+                        else:
+                                args = []
+                        args.append('-m')
+                        args.append(str(i))
+                        if numa_args:
+                                polluters.append(Executable('numactl', args))
+                        else:
+                                polluters.append(Executable('memthrash', args))
+
+        if numa_args:
+                args = copy.deepcopy(numa_args)
+                args.append('cache_cost')
+        else:
+                args = []
+        args.append('-m')
+        args.append(str(ncpu))
+        args.append('-c')
+        args.append(str(nsamples))
+        args.append('-s')
+        args.append(str(wss))
+        args.append('-w')
+        args.append(str(wcycle))
+
+        if do_huge_pages:
+                args.append('-h')
+
+        if do_uncache_pages:
+                args.append('-u')
+
+        # cache_cost wants time in microseconds
+        args.append('-x')
+        args.append(str(sleep_range_ms[0]*1000))
+        args.append('-y')
+        args.append(str(sleep_range_ms[1]*1000))
+
+        if walk == 'rand':
+                args.append('-r')
+
+        args.append('-o')
+        args.append(filename)
+
+        if numa_args:
+                probe = Executable('numactl', args)
+        else:
+                probe = Executable('cache_cost', args)
+
+        if not test_mode:
+                if do_pollute:
+                        for p in polluters:
+                                p.execute()
+                        time.sleep(10)
+
+                probe.execute()
+                probe.wait()
+
+                for p in polluters:
+                        p.kill()
+                for p in polluters:
+                        p.wait(False)
+        else:
+                print 'Process commands and arguments: '
+                if do_pollute:
+                        for p in polluters:
+                                print str(p)
+                print str(probe)
+
+def main(argv):
+        nsamples = 5000
+
+        # We may need to test different NUMA node configurations
+        # according to memory interleaving across the NUMA topology.
+        #
+        # For non-numa systems, do "<#cpu>: []"
+        # Ex., for a 4-CPU non-numa system: "4: []"
+        #
+        # NOTE: Must update NUMA hack in run_exp() for memthrash
+        # for your configuration.
+        #
+        # todo: configure numa args for system automatically
+        ncpu_and_numa_args = {
+                6: ['--cpunodebind=0', '--interleave=0']
+#               12: ['--cpunodebind=0,1', '--interleave=0,1']
+        }
+
+        wss_kb = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 3072, 4096, 8192, 12288, 16384]
+        write_cycle = [0, 64, 16, 4, 2, 1]
+
+        sleep_range_ms = [0,50]
+
+#       uncache = [False, True]
+        uncache = [False]
+        huge_pages = [False]
+
+        pollute = [False, True]
+        walk = ['seq', 'rand']
+#       walk = ['seq']
+
+        for ncpu, numa_args in ncpu_and_numa_args.iteritems():
+                for u in uncache:
+                        for h in huge_pages:
+                                if u == True and h == True:
+                                        # skip invalid combo
+                                        continue
+                                for wcycle in write_cycle:
+                                        for w in walk:
+                                                for p in pollute:
+                                                        for wss in wss_kb:
+                                                                run_exp(nsamples, ncpu, numa_args, wss, wcycle, sleep_range_ms, w, p, h, u)
+
+        if email_notification:
+                _subject = "Cache Ovh Collection Complete!"
+                _to = "gelliott@cs.unc.edu"
+                _from = "gelliott@bonham.cs.unc.edu"
+                _text = "Cache Ovh Collection Complete!"
+                _body = string.join(("From: %s" % _from, "To: %s" % _to, "Subject: %s" % _subject, "", _text), "\r\n")
+                s = smtplib.SMTP("localhost")
+                s.sendmail(_from, [_to], _body)
+                s.quit()
+
+
+
+if __name__ == "__main__":
+
+        os.environ['PATH'] += ':../liblitmus'
+        os.environ['PATH'] += ':../cache-ovh'
+
+        if 'LD_LIBRARY_PATH' not in os.environ:
+                os.environ['LD_LIBRARY_PATH'] = ''
+        os.environ['LD_LIBRARY_PATH'] += ':../liblitmus'
+
+        main(sys.argv[1:])
diff --git a/distill_pmo.py b/distill_pmo.py
new file mode 100755
index 0000000..9821dd5
--- /dev/null
+++ b/distill_pmo.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+
+import os
+import re
+import fnmatch
+import shutil as sh
+import sys
+import csv
+import numpy as np
+from scipy.stats import scoreatpercentile
+import bisect
+from optparse import OptionParser
+
+from utils.machines import machines
+
+import utils.iqr
+
+class Topology:
+        ncpus, root, leaves, dist_mat = 0, None, None, None
+        levels = ['L1', 'L2', 'L3', 'Mem', 'System']
+
+        class Node:
+                idx, name, parent, children = 0, 'Unk', None, None
+                def __init__(self, idx, name, parent = None):
+                        self.idx = idx
+                        self.name = name
+                        self.parent = parent
+                        self.children = []
+                def __repr__(self):
+                        return self.name + '_' + str(self.idx)
+
+        def __build_level_above(self, machine, l, child_nodes):
+                key = 'n' + l
+                if key in machine:
+                        cluster_sz = machine[key]
+                else:
+                        cluster_sz = 1
+                nchildren = len(child_nodes)
+                nodes = [self.Node(idx, l) for idx in range(nchildren/cluster_sz)]
+                for i in range(len(child_nodes)):
+                        child_nodes[i].parent = nodes[i/cluster_sz]
+                        nodes[i/cluster_sz].children.append(child_nodes[i])
+                return nodes
+
+        def __find_dist(self, a, b):
+                if a != b:
+                        # pass-through (ex. as CPU is to private L1)
+                        if len(a.parent.children) == 1:
+                                return self.__find_dist(a.parent, b.parent)
+                        else:
+                                return 1 + self.__find_dist(a.parent, b.parent)
+                return 0
+
+        def __build_dist_matrix(self):
+                dist_mat = np.empty([self.ncpus, self.ncpus], int)
+                for i in range(self.ncpus):
+                        for j in range(i, self.ncpus):
+                                dist_mat[i,j] = dist_mat[j,i] = self.__find_dist(self.leaves[i], self.leaves[j])
+                return dist_mat
+
+        def __init__(self, machine):
+                self.ncpus = machine['sockets']*machine['cores_per_socket']
+
+                # build the Topology bottom up
+                self.leaves = [self.Node(idx, 'CPU') for idx in range(self.ncpus)]
+                nodes = self.leaves
+                for l in self.levels:
+                        nodes = self.__build_level_above(machine, l, nodes)
+                self.root = nodes
+
+                self.dist_mat = self.__build_dist_matrix()
+
+
+        def __repr_level(self, node, stem, buf):
+                spacing = 3
+                buf += stem + node.name + '_' + str(node.idx) + '\n'
+                for c in node.children:
+                        buf = self.__repr_level(c, stem + ' '*spacing, buf)
+                return buf
+
+        def __repr__(self):
+                buf = self.__repr_level(self.root[0], '', '')
+                return buf
+
+        def distance(self, a, b):
+                return self.dist_mat[a,b]
+
+
+topologies = {}
+def get_topo(host):
+        if host in topologies:
+                return topologies[host]
+        else:
+                topo = Topology(machines[host])
+                topologies[host] = topo
+                return topo
+
+# find the max/avg/std of preemption and migration
+def process_cpmd(csv_file, params):
+
+        if 'pmo' not in params:
+                raise Exception(('not cpmd overhead file: %s)') % csv_file)
+
+        topo = get_topo(params['host'])
+
+        print 'processing ' + csv_file
+
+        ifile = open(csv_file, "r")
+        reader = csv.reader(ifile)
+        costs = {}
+
+        SAMPLE = 0
+        WRITE_CYCLE = 1
+        WSS = 2
+        DELAY = 3
+        LAST_CPU = 4
+        NEXT_CPU = 5
+        DIST = 6
+        COLD = 7
+        HOT1 = 8
+        HOT2 = 9
+        HOT3 = 10
+        AFTER_RESUME = 11
+
+        for row in reader:
+                hot = min(int(row[HOT1]), int(row[HOT2]), int(row[HOT3]))
+                after = int(row[AFTER_RESUME])
+                cost = max(after - hot, 0)
+                distance = topo.distance(int(row[NEXT_CPU]), int(row[LAST_CPU]))
+                assert distance == int(row[DIST])
+                if distance not in costs:
+                        costs[distance] = []
+                costs[distance].append(cost)
+
+        for d,c in costs.iteritems():
+                arr = np.array(c, float)
+                arr = np.sort(arr)
+                (arr, mincut, maxcut) = utils.iqr.apply_iqr(arr, 1.5)
+                for x in np.nditer(arr, op_flags=['readwrite']):
+                        x[...] = utils.machines.cycles_to_us(params['host'], x)
+                costs[d] = arr
+
+        stats = {}
+#       print costs
+        for d,arr in costs.iteritems():
+                stats[d] = {'max':arr.max(), 'median':np.median(arr), 'mean':arr.mean(), 'std':arr.std()}
+
+        return stats
+
+def parse_args():
+        parser = OptionParser("usage: %prog [files...]")
+        return parser.parse_args()
+
+def safe_split(t, delim):
+        t = t.split(delim)
+        if len(t) == 1:
+                t = tuple([t[0], None])
+        return t
+
+def get_level(machine, ncpus):
+        dist = get_topo(machine).distance(0, int(ncpus)-1)
+        names = ['L1', 'L2', 'L3', 'mem', 'sys']
+        if dist <= len(names):
+                return names[dist]
+        else:
+                raise Exception("Unable to determine level.")
+        return ''
+
+def main():
+        opts, args = parse_args()
+
+        files = filter(os.path.exists, args)
+
+        regex = fnmatch.translate("pmo_*.csv")
+        csvs = re.compile(regex)
+        files = filter(csvs.search, files)
+
+        results = {}
+        for f in files:
+                temp = os.path.basename(f).split(".csv")[0]
+                tokens = temp.split("_")
+
+                params = {k:v for (k,v) in map(lambda x: safe_split(x, "="), tokens)}
+                common = tuple([params['host'], params['ncpu'], params['wcycle'], params['polluters'], params['walk'], params['hpages'], params['upages']])
+                if common not in results:
+                        results[common] = {}
+                results[common][int(params['wss'])] = process_cpmd(f, params)
+
+#       print results
+        for common in results:
+                trends = results[common]
+                name = 'dpmo_host=%s_lvl=%s_wcycle=%s_polluters=%s_walk=%s_hpages=%s_upages=%s.csv' %
+                        (common[0], get_level(common[0], common[1]), common[2], common[3], common[4], common[5], common[6])
+                f = open(name, 'w')
+                for w,stats in iter(sorted(trends.iteritems())):
+                        f.write('%d' % w)
+                        _mean = 0
+                        _max = 0
+                        for i,data in iter(sorted(stats.iteritems())):
+                                dist_mean = data['mean']
+                                _mean = max(_mean, dist_mean)
+                                f.write(', %.6f' % dist_mean)
+                        f.write(', %.6f' % _mean)
+                        for i,data in iter(sorted(stats.iteritems())):
+                                dist_max = data['max']
+                                _max = max(_max, dist_max)
+                                f.write(', %.6f' % dist_max)
+                        f.write(', %.6f\n' % _max)
+
+if __name__ == '__main__':
+        main()
diff --git a/produce_consume_cost.py b/produce_consume_cost.py
new file mode 100755
index 0000000..fd08a8d
--- /dev/null
+++ b/produce_consume_cost.py
@@ -0,0 +1,172 @@
+#!/usr/bin/python
+
+import os
+import copy
+import sys
+import string
+import smtplib
+import socket
+import time
+import itertools
+import multiprocessing
+
+from run.executable.executable import Executable
+
+test_mode = False
+email_notification = True
+
+def run_exp(nsamples, ncpu, numa_args, wss, sleep_range_ms, walk, do_pollute, do_huge_pages, do_uncache_pages):
+
+        fstub = 'pco'
+        fstub += '_host='+socket.gethostname().split('.')[0]
+        fstub += '_ncpu='+str(ncpu)
+        fstub += '_wss='+str(wss)
+        fstub += '_smin='+str(sleep_range_ms[0]*1000)
+        fstub += '_smax='+str(sleep_range_ms[1]*1000)
+        fstub += '_polluters='+str(do_pollute)
+        fstub += '_hpages='+str(do_huge_pages)
+        fstub += '_upages='+str(do_uncache_pages)
+        fstub += '_walk='+walk
+
+        filename = fstub + '.csv'
+
+        if os.path.exists(filename):
+                return
+
+        polluters = []
+        if do_pollute:
+                for i in range(multiprocessing.cpu_count()):
+                        if numa_args:
+                                # number of CPUs in one of two NUMA nodes
+                                if ncpu == 6:
+                                        if i < 6:
+                                                numa_hack = ['--cpunodebind=0', '--interleave=0']
+                                        else:
+                                                numa_hack = ['--cpunodebind=1', '--interleave=1']
+                                else:
+                                        numa_hack = ['--cpunodebind=0,1', '--interleave=0,1']
+                                args = copy.deepcopy(numa_hack)
+                                args.append('memthrash')
+                        else:
+                                args = []
+                        args.append('-m')
+                        args.append(str(i))
+                        if numa_args:
+                                polluters.append(Executable('numactl', args))
+                        else:
+                                polluters.append(Executable('memthrash', args))
+
+        if numa_args:
+                args = copy.deepcopy(numa_args)
+                args.append('produce_consume_cost')
+        else:
+                args = []
+        args.append('-m')
+        args.append(str(ncpu))
+        args.append('-c')
+        args.append(str(nsamples))
+        args.append('-s')
+        args.append(str(wss))
+
+        if do_huge_pages:
+                args.append('-h')
+        
+        if do_uncache_pages:
+                args.append('-u')
+
+        # cache_cost wants time in microseconds
+        args.append('-x')
+        args.append(str(sleep_range_ms[0]*1000))
+        args.append('-y')
+        args.append(str(sleep_range_ms[1]*1000))
+
+        if walk == 'rand':
+                args.append('-r')
+
+        args.append('-o')
+        args.append(filename)
+
+        if numa_args:
+                probe = Executable('numactl', args)
+        else:
+                probe = Executable('produce_consume_cost', args)
+
+        if not test_mode:
+                if do_pollute:
+                        for p in polluters:
+                                p.execute()
+                        time.sleep(10)
+
+                probe.execute()
+                probe.wait()
+
+                for p in polluters:
+                        p.kill()
+                for p in polluters:
+                        p.wait(False)
+        else:
+                print 'Process commands and arguments: '
+                if do_pollute:
+                        for p in polluters:
+                                print str(p)
+                print str(probe)
+
+def main(argv):
+        nsamples = 5000
+
+        # We may need to test different NUMA node configurations
+        # according to memory interleaving across the NUMA topology.
+        #
+        # For non-numa systems, do "<#cpu>: []"
+        # Ex., for a 4-CPU non-numa system: "4: []"
+        #
+        # NOTE: Must update NUMA hack in run_exp() for memthrash
+        # for your configuration.
+        #
+        # todo: configure numa args for system automatically
+        ncpu_and_numa_args = {
+                6: ['--cpunodebind=0', '--interleave=0']
+#               12: ['--cpunodebind=0,1', '--interleave=0,1']
+        }
+
+        wss_kb = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 3072, 4096, 8192, 12288, 16384]
+        sleep_range_ms = [0,50]
+
+#       uncache = [False, True]
+        uncache = [False]
+        huge_pages = [False]
+
+        pollute = [False, True]
+#       walk = ['seq', 'rand']
+        walk = ['seq']
+
+        for ncpu, numa_args in ncpu_and_numa_args.iteritems():
+                for u in uncache:
+                        for h in huge_pages:
+                                for w in walk:
+                                        for p in pollute:
+                                                for wss in wss_kb:
+                                                        run_exp(nsamples, ncpu, numa_args, wss, sleep_range_ms, w, p, h, u)
+
+        if email_notification:
+                _subject = "Producer/Consumer Ovh Collection Complete!"
+                _to = "gelliott@cs.unc.edu"
+                _from = "gelliott@bonham.cs.unc.edu"
+                _text = "Producer/Consumer Ovh Collection Complete!"
+                _body = string.join(("From: %s" % _from, "To: %s" % _to, "Subject: %s" % _subject, "", _text), "\r\n")
+                s = smtplib.SMTP("localhost")
+                s.sendmail(_from, [_to], _body)
+                s.quit()
+
+
+
+if __name__ == "__main__":
+
+        os.environ['PATH'] += ':../liblitmus'
+        os.environ['PATH'] += ':../cache-ovh'
+
+        if 'LD_LIBRARY_PATH' not in os.environ:
+                os.environ['LD_LIBRARY_PATH'] = ''
+        os.environ['LD_LIBRARY_PATH'] += ':../liblitmus'
+
+        main(sys.argv[1:])
diff --git a/utils/__init__.py b/utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/utils/__init__.py
diff --git a/utils/iqr.py b/utils/iqr.py
new file mode 100755
index 0000000..bbecdfa
--- /dev/null
+++ b/utils/iqr.py
@@ -0,0 +1,32 @@
+import numpy as np
+from scipy.stats import scoreatpercentile
+import bisect
+
+def find_lt(a, x):
+        i = bisect.bisect_left(a,x)
+        if i:
+                return i - 1
+        else:
+                return None
+
+def find_gt(a, x):
+        i = bisect.bisect_right(a, x)
+        if i != len(a):
+                return i
+        else:
+                return None
+
+def apply_iqr(seq, extent = 1.5):
+        q1 = scoreatpercentile(seq, 25)
+        q3 = scoreatpercentile(seq, 75)
+        iqr = q3 - q1
+        start = 0
+        end = len(seq) - 1
+        l = find_lt(seq, q1 - extent*iqr)
+        if l is not None:
+                start = l + 1
+        r = find_gt(seq, q3 + extent*iqr)
+        if r is not None:
+                end = r - 1
+        seq = seq[start:end+1]
+        return (seq, q1 - extent*iqr, q3 + extent*iqr)
diff --git a/utils/machines.py b/utils/machines.py
new file mode 100644
index 0000000..37a3c2f
--- /dev/null
+++ b/utils/machines.py
@@ -0,0 +1,43 @@
+machines = {
+        'ludwig': {
+        'cpu': 2133.0,   # mhz
+        'sockets': 4,
+        'cores_per_socket': 6,
+        'L1': 256,       # kb
+        'nL1': 1,   # private
+        'L2': 3*1024,
+        'nL2': 2,   # shared by two L1 modules
+        'L3': 12*1024,
+        'nL3': 3,   # shared by three L2 modules
+        'nMem': 4   # shared by four L3 modules
+        },
+        'bonham': {
+        'cpu': 2666.0,   # mhz
+        'sockets': 2,
+        'cores_per_socket': 6,
+        'L1': 32,        # kb
+        'nL1': 1,   # private
+        'L2': 2*1024,
+        'nL2': 1,   # private
+        'L3': 12*1024,
+        'nL3': 6,   # shared by six L2 modules
+        'nMem': 1   # shared by one L3 module (per mem)
+        },
+        'ringo': {
+        'cpu': 1400.0,   # mhz
+        'sockets': 1,
+        'cores_per_socket': 4,
+        'L1': 32,        # kb (just d-cache)
+        'nL1': 1,   # private
+        'L2': 1024,
+        'nL2': 4,   # shared by four cpus
+        'L3': 0,    # n/a
+        'nL3': 0,   # n/a
+        'nMem': 1
+        }
+}
+
+def cycles_to_us(machine_name, ncycles):
+        mhz = machines[machine_name]['cpu']
+        us = ncycles / mhz
+        return us