Merge branch 'sd-vbs' of ssh://rtsrv.cs.unc.edu/public/mc2-scripts-and-benchmarks into sd-vbs

4 files changed, 379 insertions, 0 deletions

diff --git a/smt_analysis/computeLCslowdown.py b/smt_analysis/computeLCslowdown.py
new file mode 100755
index 0000000..bcd22da
--- /dev/null
+++ b/smt_analysis/computeLCslowdown.py
@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+import numpy as np
+import sys
+import plotille.plotille as plt
+from libSMT import *
+TIMING_ERROR = 1000 #ns
+ASYNC_FORMAT = False
+
+def print_usage():
+    print("This program takes in the all-pairs and baseline SMT data and computes the worst-case slowdown against any other task when SMT is enabled.", file=sys.stderr)
+    print("Level-A/B usage: {} <file -A> <file -B> <baseline file>".format(sys.argv[0]), file=sys.stderr)
+    print("Level-C usage: {} <continuous pairs> <baseline file>".format(sys.argv[0]), file=sys.stderr)
+
+# Check that we got the right number of parameters
+if len(sys.argv) < 3:
+    print_usage()
+    exit()
+
+if len(sys.argv) > 3:
+    print("Reading file using synchronous pair format...")
+    print("Are you sure you want to do this? For the RTAS'21 paper, L-A/-B pairs should use the other script.")
+    input("Press enter to continue, Ctrl+C to exit...")
+else:
+    print("Reading file using asynchronous pair format...")
+    ASYNC_FORMAT = True
+
+assert_valid_input_files(sys.argv[1:-1], print_usage)
+
+# Pull in the data
+if not ASYNC_FORMAT:
+    baseline_times, baseline_sample_cnt, baseline_max_times = load_baseline(sys.argv[3])
+    paired_times, paired_offsets, name_to_idx, idx_to_name = load_paired(sys.argv[1], sys.argv[2], len(list(baseline_times.keys())))
+    for key in baseline_times:
+        print(key,max(baseline_times[key]))
+else:
+    baseline_times, baseline_sample_cnt, baseline_max_times = load_baseline(sys.argv[2])
+    paired_times, name_to_idx, idx_to_name = load_fake_paired(sys.argv[1])
+
+# We work iff the baseline was run for the same set of benchmarks as the pairs were
+assert_base_and_pair_keys_match(baseline_times, name_to_idx)
+
+# Only consider benchmarks that are at least an order of magnitude longer than the timing error
+reliableNames = []
+for i in range(0, len(name_to_idx)):
+    benchmark = idx_to_name[i]
+    if min(baseline_times[benchmark]) > TIMING_ERROR * 10:
+        reliableNames.append(benchmark)
+
+# Compute worst-case SMT slowdown for each benchmark
+print("Bench           Mi")
+# Print rows
+sample_f = np.mean # Change this to np.mean to use mean values in Mi generation
+M_vals = []
+for b1 in reliableNames:
+    print("{:<14.14}:".format(b1), end=" ")
+    max_mi = 0
+    # Scan through everyone we ran against and find our maximum slowdown
+    for b2 in reliableNames:
+        time_with_smt = sample_f(paired_times[name_to_idx[b1]][name_to_idx[b2]])
+        time_wout_smt = sample_f(baseline_times[b1])
+        M = time_with_smt / time_wout_smt
+        max_mi = max(max_mi, M)
+    print("{:>10.3}".format(max_mi), end=" ")
+    M_vals.append(max_mi)
+    print("")
+# Print some statistics about the distribution
+print("Average: {:>5.3} with standard deviation {:>5.3} using `{}`".format(np.mean(M_vals), np.std(M_vals), sample_f.__name__))
+Ms = np.asarray(M_vals, dtype=np.float32)
+print(np.sum(Ms <= 1), "of", len(M_vals), "M_i values are at most one -", 100*np.sum(Ms <= 1)/len(M_vals), "percent")
+print(np.sum(Ms > 2), "of", len(M_vals), "M_i values are greater than two -", 100*np.sum(Ms > 2)/len(M_vals), "percent")
+M_vals_to_plot = Ms
+
+print(plt.hist(M_vals_to_plot, bins=10))
diff --git a/smt_analysis/computeSMTslowdown.py b/smt_analysis/computeSMTslowdown.py
new file mode 100755
index 0000000..805def1
--- /dev/null
+++ b/smt_analysis/computeSMTslowdown.py
@@ -0,0 +1,155 @@
+#!/usr/bin/python3
+from typing import List, Any
+import numpy as np
+from scipy import stats
+import sys
+import plotille.plotille as plt
+TIMING_ERROR = 1000 #ns
+LEVEL_C_ANALYSIS = False
+from libSMT import *
+
+def print_usage():
+    print("This program takes in the all-pairs and baseline SMT data and computes how much each program is slowed when SMT in enabled.", file=sys.stderr)
+    print("Level-A/B usage: {} <file -A> <file -B> <baseline file> --cij".format(sys.argv[0]), file=sys.stderr)
+    print("Level-C usage: {} <continuous pairs> <baseline file>".format(sys.argv[0]), file=sys.stderr)
+
+# Check that we got the right number of parameters
+if len(sys.argv) < 3:
+    print_usage()
+    exit()
+
+if len(sys.argv) > 3:
+    print("Analyzing results using Level-A/B methodology...")
+else:
+    print("Analyzing results using Level-C methodology...")
+    LEVEL_C_ANALYSIS = True
+
+assert_valid_input_files(sys.argv[1:-1], print_usage);
+
+# Print Cij values rather than Mij
+TIMES_ONLY = len(sys.argv) > 4 and "--cij" in sys.argv[4]
+OK_PAIRS_ONLY = len(sys.argv) > 4 and "--cij-ok" in sys.argv[4]
+
+# Pull in the data
+if not LEVEL_C_ANALYSIS:
+    baseline_times, baseline_sample_cnt, baseline_max_times = load_baseline(sys.argv[3])
+    paired_times, paired_offsets, name_to_idx, idx_to_name = load_paired(sys.argv[1], sys.argv[2], len(list(baseline_times.keys())))
+    for key in baseline_times:
+        print(key,max(baseline_times[key]))
+else:
+    # Paired times use an abuse of the baseline file format
+    baseline_times, baseline_sample_cnt, baseline_max_times = load_baseline(sys.argv[2])
+    paired_times, name_to_idx, idx_to_name = load_fake_paired(sys.argv[1])
+
+# We work iff the baseline was run for the same set of benchmarks as the pairs were
+assert_base_and_pair_keys_match(baseline_times, name_to_idx)
+
+# Only consider benchmarks that are at least an order of magnitude longer than the timing error
+reliableNames = []
+for i in range(0, len(name_to_idx)):
+    benchmark = idx_to_name[i]
+    if min(baseline_times[benchmark]) > TIMING_ERROR * 10:
+        reliableNames.append(benchmark)
+
+# Compute SMT slowdown for each benchmark
+# Output format: table, each row is one benchmark and each column is one benchmark
+#                each cell is base1 + base2*m = pair solved for m, aka (pair - base1) / base2
+# Print table header
+print("Bench          ", end=" ")
+for name in reliableNames:
+    if not TIMES_ONLY: print("{:<10.10}".format(name), end=" ")
+    if TIMES_ONLY: print("{:<12.12}".format(name), end=" ")
+print()
+# Print rows
+sample_f = max # Change this to np.mean to use mean values in Mij generation
+M_vals = []
+for b1 in reliableNames:
+    if not TIMES_ONLY: print("{:<14.14}:".format(b1), end=" ")
+    if TIMES_ONLY: print("{:<14.14}:".format(b1), end=" ")
+    for b2 in reliableNames:
+        if not LEVEL_C_ANALYSIS:
+            Ci = max(sample_f(baseline_times[b1]), sample_f(baseline_times[b2]))
+            Cj = min(sample_f(baseline_times[b1]), sample_f(baseline_times[b2]))
+            Cij = sample_f(paired_times[name_to_idx[b1]][name_to_idx[b2]])
+            if False:
+                M = np.std(paired_times[name_to_idx[b1]][name_to_idx[b2]]) / np.mean(paired_times[name_to_idx[b1]][name_to_idx[b2]])
+            else:
+                M = (Cij - Ci) / Cj
+            if Cij and Cj * 10 > Ci: # We don't pair tasks with more than a 10x difference in length
+                M_vals.append(M)
+                if not TIMES_ONLY: print("{:>10.3}".format(M), end=" ")
+            else:
+                if not TIMES_ONLY: print("{:>10}".format("N/A"), end=" ")
+
+            if TIMES_ONLY and (not OK_PAIRS_ONLY or Cj * 10 > Ci):
+                print("{:>12}".format(Cij), end=" ")
+            elif OK_PAIRS_ONLY and Cj * 10 <= Ci:
+                print("{:>12}".format("0"), end=" ")
+
+        else:
+            time_with_smt = sample_f(paired_times[name_to_idx[b1]][name_to_idx[b2]])
+            time_wout_smt = sample_f(baseline_times[b1])
+            M = time_with_smt / time_wout_smt
+            M_vals.append(M)
+            print("{:>10.3}".format(M), end=" ")
+    print("")
+# Print some statistics about the distribution
+print("Average: {:>5.3} with standard deviation {:>5.3} using `{}`".format(np.mean(M_vals), np.std(M_vals), sample_f.__name__))
+Ms = np.asarray(M_vals, dtype=np.float32)
+if not LEVEL_C_ANALYSIS:
+    print(np.sum(Ms <= 0), "of", len(M_vals), "M_i:j values are at most zero -", 100*np.sum(Ms <= 0)/len(M_vals), "percent")
+    print(np.sum(Ms > 1), "of", len(M_vals), "M_i:j values are greater than one -", 100*np.sum(Ms > 1)/len(M_vals), "percent")
+    M_vals_to_plot = Ms[np.logical_and(Ms > 0, Ms <= 1)]
+else:
+    print(np.sum(Ms <= 1), "of", len(M_vals), "M_i:j values are at most one -", 100*np.sum(Ms <= 1)/len(M_vals), "percent")
+    print(np.sum(Ms > 2), "of", len(M_vals), "M_i:j values are greater than two -", 100*np.sum(Ms > 2)/len(M_vals), "percent")
+    M_vals_to_plot = Ms
+
+print("Using Sim's analysis, average: {:>5.3} with standard deviation {:>5.3} using `{}`".format(np.mean(list(M_vals_to_plot)), np.std(list(M_vals_to_plot)), sample_f.__name__))
+print(plt.hist(M_vals_to_plot, bins=10))
+