Fixed bug in trace_reader.py and implemented unit test in response.

trace_reader.py was losing records (1 per CPU) so I created
naive_trace_reader.py which reads records in the simplest way
possible but does not sort them. The unit test ensures that both
readers get the same number of records.

3 files changed, 179 insertions, 1 deletions

diff --git a/naive_trace_reader.py b/naive_trace_reader.py
new file mode 100644
index 0000000..9a59353
--- /dev/null
+++ b/naive_trace_reader.py
@@ -0,0 +1,165 @@
+###############################################################################
+# Description
+###############################################################################
+
+# trace_reader(files) returns an iterator which produces records
+# OUT OF ORDER from the files given. (the param is a list of files.)
+#
+# The non-naive trace_reader has a lot of complex logic which attempts to
+# produce records in order (even though they are being pulled from multiple
+# files which themselves are only approximately ordered). This trace_reader
+# attempts to be as simple as possible and is used in the unit tests to
+# make sure the total number of records read by the normal trace_reader is
+# the same as the number of records read by this one.
+
+###############################################################################
+# Imports
+###############################################################################
+
+import struct
+
+
+###############################################################################
+# Public functions
+###############################################################################
+
+# Generator function returning an iterable over records in a trace file.
+def trace_reader(files):
+    for file in files:
+        with open(file,'rb') as f:
+            while True:
+                data = f.read(RECORD_HEAD_SIZE)
+                try:
+                    type_num = struct.unpack_from('b',data)[0]
+                except struct.error: 
+                    break #We read to the end of the file
+                type = _get_type(type_num)
+                try:
+                    values = struct.unpack_from(StHeader.format + 
+                        type.format,data)
+                    record_dict = dict(zip(type.keys,values))
+                except struct.error:
+                    f.close()
+                    print("Invalid record detected, stopping.")
+                    exit()
+
+                # Convert the record_dict into an object
+                record = _dict2obj(record_dict)
+
+                # Give it a type name (easier to work with than type number)
+                record.type_name = _get_type_name(type_num)
+
+                # All records should have a 'record type' field.
+                # e.g. these are 'event's as opposed to 'error's
+                record.record_type = "event"
+
+                # If there is no timestamp, set the time to 0
+                if 'when' not in record.__dict__.keys():
+                    record.when = 0
+
+                yield record
+
+###############################################################################
+# Private functions
+###############################################################################
+
+# Convert a dict into an object
+def _dict2obj(d):
+    class Obj: pass
+    o = Obj()
+    for key in d.keys():
+        o.__dict__[key] = d[key]
+    return o 
+
+###############################################################################
+# Trace record data types and accessor functions
+###############################################################################
+
+# Each class below represents a type of event record. The format attribute
+# specifies how to decode the binary record and the keys attribute
+# specifies how to name the pieces of information decoded. Note that all
+# event records have a common initial 24 bytes, represented by the StHeader
+# class.
+
+RECORD_HEAD_SIZE = 24
+
+class StHeader:
+    format =  '<bbhi'
+    formatStr = struct.Struct(format)
+    keys = ['type','cpu','pid','job']
+    message = 'The header.'
+
+class StNameData:
+    format =  '16s'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['name']
+    message = 'The name of the executable of this process.'
+
+class StParamData:  
+    format =  'IIIc'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['wcet','period','phase','partition']
+    message = 'Regular parameters.'
+
+class StReleaseData:
+    format =  'QQ'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when','deadline']
+    message = 'A job was/is going to be released.'
+
+#Not yet used by Sched Trace 
+class StAssignedData:
+    format =  'Qc'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when','target']
+    message = 'A job was assigned to a CPU.'
+
+class StSwitchToData:
+    format =  'QI'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when','exec_time']
+    message = 'A process was switched to on a given CPU.'
+
+class StSwitchAwayData:
+    format =  'QI'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when','exec_time']
+    message = 'A process was switched away on a given CPU.'
+
+class StCompletionData:
+    format =  'Q3x?c'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when','forced?','flags']
+    message = 'A job completed.'
+
+class StBlockData:
+    format =  'Q'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when']
+    message = 'A task blocks.'
+
+class StResumeData:
+    format =  'Q'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when']
+    message = 'A task resumes.'
+
+class StSysReleaseData:
+    format =  'QQ'
+    formatStr = struct.Struct(StHeader.format + format)
+    keys = StHeader.keys + ['when','release']
+    message = 'All tasks have checked in, task system released by user'
+
+# Return the binary data type, given the type_num
+def _get_type(type_num):
+    types = [None,StNameData,StParamData,StReleaseData,StAssignedData,
+             StSwitchToData,StSwitchAwayData,StCompletionData,StBlockData,
+             StResumeData,StSysReleaseData]
+    return types[type_num]
+
+# Return the type name, given the type_num (this is simply a convenience to
+#     programmers of other modules)
+def _get_type_name(type_num):
+    type_names = [None,"name","params","release","assign","switch_to",
+        "switch_away","completion","block","resume","sys_release"]
+    return type_names[type_num]
diff --git a/runtests.py b/runtests.py
index f4a9274..1ae29f4 100755
--- a/runtests.py
+++ b/runtests.py
@@ -12,6 +12,7 @@
 ###############################################################################
 
 import trace_reader
+import naive_trace_reader
 import os
 
 ###############################################################################
@@ -39,4 +40,16 @@ def test1():
         last_time = item.when
     return "[SUCCESS]"
 
+# Does our fancy trace reader get the same number of files as our naive one?
+# (See naive_trace_reader.py for further explanation)
+def test2():
+    stream = trace_reader.trace_reader(files)
+    num_records = len(list(stream))
+    stream = naive_trace_reader.trace_reader(files)
+    naive_num_records = len(list(stream))
+    if num_records != naive_num_records:
+        return "[FAIL]"
+    return "[SUCCESS]"
+
 print("Test 1: {}".format(test1()))
+print("Test 2: {}".format(test2()))
diff --git a/trace_reader.py b/trace_reader.py
index 7b579b6..31f1812 100644
--- a/trace_reader.py
+++ b/trace_reader.py
@@ -40,7 +40,7 @@ def trace_reader(files):
     file_iter_buff = [] # file iterator buffers
     for file in files:
         file_iter = _get_file_iter(file)
-        file_iters.append(_get_file_iter(file))
+        file_iters.append(file_iter)
         file_iter_buff.append([next(file_iter)])
 
     # We keep 100 records in each buffer and then keep the buffer sorted