From 14a40b99735f09f6e70b8e897acbb622f9115ca3 Mon Sep 17 00:00:00 2001 From: Mac Mollison Date: Sat, 13 Mar 2010 12:09:00 -0500 Subject: Directory restructuring Reworked directory (and modified some code a bit) to match intended architecture. --- viz/trace_reader.py | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 viz/trace_reader.py (limited to 'viz/trace_reader.py') diff --git a/viz/trace_reader.py b/viz/trace_reader.py new file mode 100644 index 0000000..a4ff964 --- /dev/null +++ b/viz/trace_reader.py @@ -0,0 +1,245 @@ +############################################################################### +# Description +############################################################################### + +# trace_reader(files) returns an iterator which produces records +# in order from the files given. (the param is a list of files.) +# +# Each record is just a Python object. It is guaranteed to have the following +# attributes: +# - 'pid': pid of the task +# - 'job': job number for that task +# - 'cpu', given by LITMUS +# - 'when', given by LITMUS as a timestamp. LITMUS does not provide a +# timestamp for all records. In this case, when is set to 0. +# - 'type', a numerical value given by LITMUS +# - 'type_name', a human-readable name defined in this module +# - 'record_type', set to 'event' by this module (to distinguish from, e.g., +# error records produced elsewhere). +# - Possible additional attributes, depending on the type of record. +# +# To find out exactly what attributes are set for each record type, look at +# the trace-parsing information at the bottom of this file. + +############################################################################### +# Imports +############################################################################### + +import struct + + +############################################################################### +# Public functions +############################################################################### + +# Generator function returning an iterable over records in a trace file. +def trace_reader(files): + + # Yield a record indicating the number of CPUs, used by the G-EDF test + class Obj: pass + record = Obj() + record.record_type = "meta" + record.type_name = "num_cpus" + record.num_cpus = len(files) + yield record + + # Create iterators for each file and a buffer to store records in + file_iters = [] # file iterators + file_iter_buff = [] # file iterator buffers + for file in files: + file_iter = _get_file_iter(file) + file_iters.append(file_iter) + file_iter_buff.append([file_iter.next()]) + + # We keep 100 records in each buffer and then keep the buffer sorted + # This is because records may have been recorded slightly out of order + # This cannot guarantee records are produced in order, but it makes it + # overwhelmingly probably. + for x in range(0,len(file_iter_buff)): + for y in range(0,100): + file_iter_buff[x].append(file_iters[x].next()) + for x in range(0,len(file_iter_buff)): + file_iter_buff[x] = sorted(file_iter_buff[x],key=lambda rec: rec.when) + + # Remember the time of the last record. This way, we can make sure records + # truly are produced in monotonically increasing order by time and terminate + # fatally if they are not. + last_time = None + + # Keep pulling records as long as we have a buffer + while len(file_iter_buff) > 0: + + # Select the earliest record from those at the heads of the buffers + earliest = -1 + buff_to_refill = -1 + for x in range(0,len(file_iter_buff)): + if earliest==-1 or file_iter_buff[x][0].when < earliest.when: + earliest = file_iter_buff[x][0] + buff_to_refill = x + + # Take it out of the buffer + del file_iter_buff[buff_to_refill][0] + + # Try to append a new record to the buffer (if there is another) and + # then keep the buffer sorted + try: + file_iter_buff[buff_to_refill].append(file_iters[buff_to_refill].next()) + file_iter_buff[buff_to_refill] = sorted(file_iter_buff[buff_to_refill], + key=lambda rec: rec.when) + + # If there aren't any more records, fine. Unless the buffer is also empty. + # If that is the case, delete the buffer. + except StopIteration: + if len(file_iter_buff[buff_to_refill]) < 1: + del file_iter_buff[buff_to_refill] + del file_iters[buff_to_refill] + + # Check for monotonically increasing time + if last_time is not None and earliest.when < last_time: + exit("FATAL ERROR: trace_reader.py: out-of-order record produced") + else: + last_time = earliest.when + + # Yield the record + yield earliest + +############################################################################### +# Private functions +############################################################################### + +# Returns an iterator to pull records from a file +def _get_file_iter(file): + f = open(file,'rb') + while True: + data = f.read(RECORD_HEAD_SIZE) + try: + type_num = struct.unpack_from('b',data)[0] + except struct.error: + break #We read to the end of the file + type = _get_type(type_num) + try: + values = struct.unpack_from(StHeader.format + + type.format,data) + record_dict = dict(zip(type.keys,values)) + except struct.error: + f.close() + print "Invalid record detected, stopping." + exit() + + # Convert the record_dict into an object + record = _dict2obj(record_dict) + + # Give it a type name (easier to work with than type number) + record.type_name = _get_type_name(type_num) + + # All records should have a 'record type' field. + # e.g. these are 'event's as opposed to 'error's + record.record_type = "event" + + # If there is no timestamp, set the time to 0 + if 'when' not in record.__dict__.keys(): + record.when = 0 + + yield record + +# Convert a dict into an object +def _dict2obj(d): + class Obj(object): pass + o = Obj() + for key in d.keys(): + o.__dict__[key] = d[key] + return o + +############################################################################### +# Trace record data types and accessor functions +############################################################################### + +# Each class below represents a type of event record. The format attribute +# specifies how to decode the binary record and the keys attribute +# specifies how to name the pieces of information decoded. Note that all +# event records have a common initial 24 bytes, represented by the StHeader +# class. + +RECORD_HEAD_SIZE = 24 + +class StHeader: + format = '