1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
#!/usr/bin/env python
import os
import re
import fnmatch
import shutil as sh
import sys
import csv
import numpy as np
from scipy.stats import scoreatpercentile
import bisect
from optparse import OptionParser
from utils.machines import machines
import utils.iqr
class Topology:
ncpus, root, leaves, dist_mat = 0, None, None, None
levels = ['L1', 'L2', 'L3', 'Mem', 'System']
class Node:
idx, name, parent, children = 0, 'Unk', None, None
def __init__(self, idx, name, parent = None):
self.idx = idx
self.name = name
self.parent = parent
self.children = []
def __repr__(self):
return self.name + '_' + str(self.idx)
def __build_level_above(self, machine, l, child_nodes):
key = 'n' + l
if key in machine:
cluster_sz = machine[key]
else:
cluster_sz = 1
nchildren = len(child_nodes)
nodes = [self.Node(idx, l) for idx in range(nchildren/cluster_sz)]
for i in range(len(child_nodes)):
child_nodes[i].parent = nodes[i/cluster_sz]
nodes[i/cluster_sz].children.append(child_nodes[i])
return nodes
def __find_dist(self, a, b):
if a != b:
# pass-through (ex. as CPU is to private L1)
if len(a.parent.children) == 1:
return self.__find_dist(a.parent, b.parent)
else:
return 1 + self.__find_dist(a.parent, b.parent)
return 0
def __build_dist_matrix(self):
dist_mat = np.empty([self.ncpus, self.ncpus], int)
for i in range(self.ncpus):
for j in range(i, self.ncpus):
dist_mat[i,j] = dist_mat[j,i] = self.__find_dist(self.leaves[i], self.leaves[j])
return dist_mat
def __init__(self, machine):
self.ncpus = machine['sockets']*machine['cores_per_socket']
# build the Topology bottom up
self.leaves = [self.Node(idx, 'CPU') for idx in range(self.ncpus)]
nodes = self.leaves
for l in self.levels:
nodes = self.__build_level_above(machine, l, nodes)
self.root = nodes
self.dist_mat = self.__build_dist_matrix()
def __repr_level(self, node, stem, buf):
spacing = 3
buf += stem + node.name + '_' + str(node.idx) + '\n'
for c in node.children:
buf = self.__repr_level(c, stem + ' '*spacing, buf)
return buf
def __repr__(self):
buf = self.__repr_level(self.root[0], '', '')
return buf
def distance(self, a, b):
return self.dist_mat[a,b]
topologies = {}
def get_topo(host):
if host in topologies:
return topologies[host]
else:
topo = Topology(machines[host])
topologies[host] = topo
return topo
def non_polluter_filename(csv_file):
return re.sub(r"polluters=True", r"polluters=False", csv_file)
# find the max/avg/std of preemption and migration
def process_cpmd(csv_file, params):
if 'pco' not in params:
raise Exception(('not producer/consumer overhead file: %s)') % csv_file)
topo = get_topo(params['host'])
print 'processing ' + csv_file
ifile = open(csv_file, "r")
bestcase = open(non_polluter_filename(csv_file), "r")
reader = csv.reader(ifile)
bc_reader = csv.reader(bestcase)
costs = {}
SAMPLE = 0
WSS = 1
DELAY = 2
LAST_CPU = 3
NEXT_CPU = 4
DIST = 5
PRODUCE_COLD = 6
PRODUCE_HOT = 7
CONSUME_COLD = 8
CONSUME_HOT = 9
for (row, bc_row) in zip(reader, bc_reader):
cold = int(row[PRODUCE_COLD])
distance = int(row[DIST])
if distance not in costs:
costs[distance] = []
costs[distance].append(cold)
for d,c in costs.iteritems():
arr = np.array(c, float)
arr = np.sort(arr)
(arr, mincut, maxcut) = utils.iqr.apply_iqr(arr, 1.5)
for x in np.nditer(arr, op_flags=['readwrite']):
x[...] = utils.machines.cycles_to_us(params['host'], x)
costs[d] = arr
stats = {}
# print costs
for d,arr in costs.iteritems():
stats[d] = {'max':arr.max(), 'median':np.median(arr), 'mean':arr.mean(), 'std':arr.std()}
return stats
def parse_args():
parser = OptionParser("usage: %prog [files...]")
return parser.parse_args()
def safe_split(t, delim):
t = t.split(delim)
if len(t) == 1:
t = tuple([t[0], None])
return t
def get_level(machine, ncpus):
dist = get_topo(machine).distance(0, int(ncpus)-1)
names = ['L1', 'L2', 'L3', 'mem', 'sys']
if dist <= len(names):
return names[dist]
else:
raise Exception("Unable to determine level.")
return ''
def main():
opts, args = parse_args()
files = filter(os.path.exists, args)
regex = fnmatch.translate("pco_*.csv")
csvs = re.compile(regex)
files = filter(csvs.search, files)
results = {}
for f in files:
temp = os.path.basename(f).split(".csv")[0]
tokens = temp.split("_")
params = {k:v for (k,v) in map(lambda x: safe_split(x, "="), tokens)}
common = tuple([params['host'], params['ncpu'], params['polluters'], params['walk'], params['hpages'], params['upages']])
if common not in results:
results[common] = {}
results[common][int(params['wss'])] = process_cpmd(f, params)
# print results
for common in results:
trends = results[common]
for t in ['max', 'median', 'mean']:
name = 'dwo_cold_host=%s_lvl=%s_polluters=%s_walk=%s_hpages=%s_upages=%s_type=%s.csv' % (common[0], get_level(common[0], common[1]), common[2], common[3], common[4], common[5], t)
f = open(name, 'w')
f.write('WSS,L1,L2,L3,MEM\n')
for w,stats in iter(sorted(trends.iteritems())):
f.write('%d' % w)
for i,data in iter(sorted(stats.iteritems())):
val = data[t]
f.write(',%.6f' % val)
f.write('\n')
if __name__ == '__main__':
main()
|
