Added progress submodule to report progress.

This submodule allows us to measure the speed at which
trace files are processed and reports on its progress as
it goes along.

2 files changed, 56 insertions, 1 deletions

diff --git a/unit_trace/progress.py b/unit_trace/progress.py
new file mode 100644
index 0000000..d987ecd
--- /dev/null
+++ b/unit_trace/progress.py
@@ -0,0 +1,46 @@
+###############################################################################
+# Description
+###############################################################################
+
+# Display progress information:
+# - Total number of bytes in trace files
+# - Total number of event records in trace files
+# - Message for every 1,000 records processed
+# - Total records processed
+# - Total elapsed time
+
+###############################################################################
+# Imports
+###############################################################################
+
+import time
+import sys
+import os
+
+###############################################################################
+# Public functions
+###############################################################################
+
+def progress(stream):
+
+    start_time = 0
+    count = 0
+    
+    for record in stream:
+        if record.record_type=="event":
+            count += 1
+        if (count % 1000) == 0 and count > 0:
+            sys.stderr.write(("Parsed %d event records\n") % (count))
+        if record.record_type=="meta" and record.type_name=="trace_files":
+            bytes = 0
+            for file in record.files:
+                bytes += int(os.path.getsize(file))
+            sys.stderr.write(("Total bytes  : %d\n") % (bytes))
+            # 192 bits per event record, 8 bits per byte
+            sys.stderr.write(("Total records: %d\n") % (bytes * 8 / 192))
+            start_time = time.time()
+        yield record
+
+    sys.stderr.write(("Total records processed: %d\n") % (count))
+    sys.stderr.write(("Time elapsed: %ds\n") % (time.time() - start_time))
+
diff --git a/unit_trace/trace_reader.py b/unit_trace/trace_reader.py
index a4ff964..44a3c75 100644
--- a/unit_trace/trace_reader.py
+++ b/unit_trace/trace_reader.py
@@ -26,6 +26,7 @@
 ###############################################################################
 
 import struct
+import sys
 
 
 ###############################################################################
@@ -35,10 +36,18 @@ import struct
 # Generator function returning an iterable over records in a trace file.
 def trace_reader(files):
 
-    # Yield a record indicating the number of CPUs, used by the G-EDF test
+    # Yield a record containing the input files
+    # This is used by progress.py to calculate progress
     class Obj: pass
     record = Obj()
     record.record_type = "meta"
+    record.type_name = "trace_files"
+    record.files = files
+    yield record
+
+    # Yield a record indicating the number of CPUs, used by the G-EDF test
+    record = Obj()
+    record.record_type = "meta"
     record.type_name = "num_cpus"
     record.num_cpus = len(files)
     yield record