Added run_exps.py option to --retry failed experiments.

If the retry flag is specified, failed experiments will be re-run after all
other experiments have run. They can be re-run at most 5 times.

This commit required a refactoring of run_exps.py to clean up the main
experiment running loop.

4 files changed, 194 insertions, 118 deletions

diff --git a/gen/edf_generators.py b/gen/edf_generators.py
index a722c21..8e4b8df 100644
--- a/gen/edf_generators.py
+++ b/gen/edf_generators.py
@@ -28,6 +28,7 @@ class EdfGenerator(gen.Generator):
         pdist = self._create_dist('period',
                                   exp_params['periods'],
                                   gen.NAMED_PERIODS)
+
         udist = self._create_dist('utilization',
                                   exp_params['utils'],
                                   gen.NAMED_UTILIZATIONS)
diff --git a/gen_exps.py b/gen_exps.py
index b847661..00ce27b 100755
--- a/gen_exps.py
+++ b/gen_exps.py
@@ -43,6 +43,14 @@ def load_file(fname):
     except:
            raise IOError("Invalid generation file: %s" % fname)
 
+def print_descriptions(described):
+    for generator in described.split(','):
+        if generator not in gen.get_generators():
+            sys.stderr.write("No generator '%s'\n" % generator)
+        else:
+            print("Generator '%s', " % generator)
+            gen.get_generators()[generator]().print_help()
+
 def main():
     opts, args = parse_args()
 
@@ -50,12 +58,7 @@ def main():
     if opts.list_gens:
         print(", ".join(gen.get_generators()))
     if opts.described != None:
-        for generator in opts.described.split(','):
-            if generator not in gen.get_generators():
-                sys.stderr.write("No generator '%s'\n" % generator)
-            else:
-                print("Generator '%s', " % generator)
-                gen.get_generators()[generator]().print_help()
+        print_descriptions(opts.described)
     if opts.list_gens or opts.described:
         return 0
 
diff --git a/run/experiment.py b/run/experiment.py
index b0e46b6..9a70414 100644
--- a/run/experiment.py
+++ b/run/experiment.py
@@ -2,6 +2,7 @@ import os
 import time
 import run.litmus_util as lu
 import shutil as sh
+
 from operator import methodcaller
 
 class ExperimentException(Exception):
@@ -69,21 +70,24 @@ class Experiment(object):
             executable.cwd = self.working_dir
         map(assign_cwd, self.executables)
 
-    def __kill_all(self):
-        if lu.waiting_tasks():
-            released = lu.release_tasks()
-            self.log("Re-released %d tasks" % released)
+    def __try_kill_all(self):
+        try:
+            if lu.waiting_tasks():
+                released = lu.release_tasks()
+                self.log("Re-released %d tasks" % released)
 
-            time.sleep(1)
+                time.sleep(1)
 
-        self.log("Killing all tasks")
-        for e in self.executables:
-            try:
-                e.kill()
-            except:
-                pass
+            self.log("Killing all tasks")
+            for e in self.executables:
+                try:
+                    e.kill()
+                except:
+                    pass
 
-        time.sleep(1)
+            time.sleep(1)
+        except:
+            self.log("Failed to kill all tasks.")
 
     def __strip_path(self, path):
         '''Shorten path to something more readable.'''
@@ -138,7 +142,7 @@ class Experiment(object):
             now_ready = lu.waiting_tasks()
             if now_ready != num_ready:
                 wait_start = time.time()
-                num_ready  = lu.now_ready
+                num_ready  = now_ready
 
     def __run_tasks(self):
         self.log("Starting %d tasks" % len(self.executables))
@@ -185,6 +189,7 @@ class Experiment(object):
 
         sched = lu.scheduler()
         if sched != "Linux":
+            self.log("Switching back to Linux scheduler")
             try:
                 lu.switch_scheduler("Linux")
             except:
@@ -229,6 +234,8 @@ class Experiment(object):
         self.log("Stopping regular tracers")
         map(methodcaller('stop_tracing'), self.regular_tracers)
 
+        os.system('sync')
+
     def log(self, msg):
         print("[Exp %s]: %s" % (self.name, msg))
 
@@ -236,6 +243,7 @@ class Experiment(object):
         self.__to_linux()
 
         succ = False
+        exception = None
         try:
             self.__setup()
 
@@ -244,16 +252,20 @@ class Experiment(object):
                 self.log("Saving results in %s" % self.finished_dir)
                 succ = True
             except Exception as e:
+                exception = e
+
                 # Give time for whatever failed to finish failing
                 time.sleep(2)
-                self.__kill_all()
 
-                raise e
-            finally:
-                self.__teardown()
+                self.__try_kill_all()
         finally:
-            self.log("Switching back to Linux scheduler")
-            self.__to_linux()
+            try:
+                self.__teardown()
+                self.__to_linux()
+            except Exception as e:
+                exception = exception or e
+            finally:
+                if exception: raise exception
 
         if succ:
             self.__save_results()
diff --git a/run_exps.py b/run_exps.py
index a15018d..1d46b45 100755
--- a/run_exps.py
+++ b/run_exps.py
@@ -3,6 +3,7 @@ from __future__ import print_function
 
 import common as com
 import os
+import pprint
 import re
 import shutil
 import sys
@@ -11,17 +12,26 @@ import run.tracer as trace
 from config.config import PARAMS,DEFAULTS
 from collections import namedtuple
 from optparse import OptionParser
+from parse.enum import Enum
 from run.executable.executable import Executable
 from run.experiment import Experiment,ExperimentDone,SystemCorrupted
 from run.proc_entry import ProcEntry
 
+'''Maximum times an experiment will be retried'''
+MAX_RETRY = 5
+
 '''Customizable experiment parameters'''
 ExpParams = namedtuple('ExpParams', ['scheduler', 'duration', 'tracers',
                                      'kernel', 'config_options', 'file_params',
                                      'pre_script', 'post_script'])
+'''Tracked with each experiment'''
+ExpState = Enum(['Failed', 'Succeeded', 'Invalid', 'Done', 'None'])
+ExpData  = com.recordtype('ExpData', ['name', 'params', 'sched_file', 'out_dir',
+                                      'retries', 'state'])
 '''Comparison of requested versus actual kernel compile parameter value'''
 ConfigResult = namedtuple('ConfigResult', ['param', 'wanted', 'actual'])