1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
from Cheetah.Template import Template
from collections import defaultdict,namedtuple
from point import SummaryPoint,Type
from dir_map import DirMap
from col_map import ColMap,ColMapBuilder
from pprint import pprint
class TupleTable(object):
def __init__(self, col_map, default=lambda:[]):
self.col_map = col_map
self.table = defaultdict(default)
def get_col_map(self):
return self.col_map
def __getitem__(self, kv):
key = self.col_map.get_key(kv)
return self.table[key]
def __setitem__(self, kv, value):
key = self.col_map.get_key(kv)
self.table[key]
def __contains__(self, kv):
key = self.col_map.get_key(kv)
return key in self.table
def __iter__(self):
return self.table.iteritems()
def reduce(self):
reduced = ReducedTupleTable(self.col_map)
for key, value in self.table.iteritems():
if type(value) == type([]):
value = SummaryPoint(value[0].id, value)
reduced.table[key] = value
return reduced
def __str__(self):
s = str(Template("""ColMap: $col_map
#for $item in $table
$item :$table[$item]
#end for""", searchList=vars(self)))
return s
class ReducedTupleTable(TupleTable):
def __init__(self, col_map):
super(ReducedTupleTable, self).__init__(col_map, default=SummaryPoint)
def __add_to_dirmap(self, dir_map, variable, kv, point):
value = kv.pop(variable)
for stat in point.get_stats():
summary = point[stat]
for summary_type in Type:
measurement = summary[summary_type]
for base_type in Type:
if not base_type in measurement:
continue
# Ex: release/num_tasks/measured-max/avg/x=5.csv
leaf = (self.col_map.encode(kv) or "line") + ".csv"
path = [ stat, variable, base_type, summary_type, leaf ]
result = measurement[base_type]
dir_map.add_values(path, [(value, result)])
kv[variable] = value
def to_dir_map(self):
dir_map = DirMap()
for key, point in self.table.iteritems():
kv = self.col_map.get_kv(key)
for col in self.col_map.columns():
val = kv[col]
try:
float(val)
except:
# Only vary numbers. Otherwise, just have seperate files
continue
self.__add_to_dirmap(dir_map, col, kv, point)
dir_map.remove_childless()
return dir_map
@staticmethod
def from_dir_map(dir_map):
Leaf = namedtuple('Leaf', ['stat', 'variable', 'base',
'summary', 'config', 'values'])
def next_type(path):
return path.pop() if path[-1] in Type else Type.Avg
def leafs():
for path, node in dir_map.leafs():
# The path will be of at least size 1: the filename
leaf = path.pop()
base = path.pop() if (path and path[-1] in Type) else Type.Avg
summ = path.pop() if (path and path[-1] in Type) else Type.Avg
path += ['?', '?'][len(path):]
[stat, variable] = path
config_str = leaf[:leaf.index('.csv')]
config = ColMap.decode(config_str)
leaf = Leaf(stat, variable, base, summ,
config, node.values)
yield leaf
builder = ColMapBuilder()
# Gather all possible config values for ColMap
for leaf_deets in leafs():
for k, v in leaf_deets.config.iteritems():
builder.try_add(k, v)
col_map = builder.build()
table = ReducedTupleTable(col_map)
# Set values at each point
for leaf in leafs():
for (x, y) in leaf.values:
leaf.config[leaf.variable] = str(x)
summary = table[leaf.config][leaf.stat]
summary[leaf.summary][leaf.base] = y
return table
def write_map(self, out_map):
rows = {}
for key, point in self.table.iteritems():
row = {}
for name,measurement in point:
name = name.lower().replace('_','-')
row[name]={}
for base_type in Type:
type_key = str(base_type).lower()
if base_type in measurement[Type.Avg]:
value = measurement[Type.Avg][base_type]
row[name][type_key] = value
rows[key] = row
result = {'columns': self.col_map.columns(), 'rows':rows}
with open(out_map, 'wc') as map_file:
pprint(result,stream=map_file, width=20)
|
