Add pair benchmarking and isolation specification into run_bench.sh

This allows `run_bench.sh` to replace all previous general paired
benchmarking scripts. See `run_everything.sh` for an example of how
to use the new script.

This also fixes two critical bugs present in the old scripts:
1. The cpus_list was incorrectly set for resctrl when running
   synchronous pair timing with L2+L3 isolation (i). Each write
   to that file replaces the previous contents, and it was previously
   written once for each core. This prevented the entire 1st core
   from using any L3 cache.
2. The competing task list was indexed improperly in the asynchronous
   pair timing script. This caused the competing task to always be
   identical to the task being profiled.
These issues combine to imply that in the rejected RTSS'20 paper:
- All Level-C SMT timing data is suspect
- Level-A and -B SMT timing data with L2+L3 isolation (i) is
  wildly optimistic

1 files changed, 204 insertions, 57 deletions

diff --git a/run_bench.sh b/run_bench.sh
index 4e09faa..250d843 100755
--- a/run_bench.sh
+++ b/run_bench.sh
@@ -1,24 +1,43 @@
 #!/bin/bash
+# Copyright 2020 Joshua Bakita
+# A unified script for timing:
+# - SMT pairs with synchronous launches (hard real-time methodolgy)
+# - SMT pairs with asynchronous launches (soft real-time methodolgy)
+# - Unpaired tasks
+# - The cache alloc vs WSS setting exploration for DIS
+# These can all be done with:
+# - No cache management
+# - L3 cache splitting
+# - L2+L3 cache splitting (if OS supported)
+
+if grep -q "#define LITMUS 1" ../extra.h ./extra.h 2> /dev/null; then
+    LITMUS=1
+fi
 
 if [ $# -lt 5 ]; then
-    echo "Usage $0 -m MODE -p CPU -l LOOPS -b FILE [-c] RUN_ID"
-    echo " -m base|dis  Which benchmarking mode to use"
+    echo "Usage $0 -m MODE -p CPU -l LOOPS -b FILE [-B] [-I] RUN_ID"
+    echo " -m base|dis|pair  Which benchmarking mode to use"
     echo " -p CPU       Which CPU to run each benchmark on"
     echo " -l LOOPS     How many loops of each benchmark to do"
     echo " -b FILE      List of benchmarks to execute. Optional tab-"
     echo "              delimited 2nd column specifies an input"
     echo "              generation command (fed to bench via stdin)"
     echo " -B           Enable background contenders on other CCXes"
+    echo " -I xi|i3|i   Isolation mode: none (xi) (default), way-based"
+    echo "              L3-only (i3), or color-based L2+L3 (i)"
     echo "Mode base requires no additional options."
     echo "Mode dis does not support the 2nd col. of -b and needs:"
     echo " -W FILE      List of working set sizes to pass gen_input.py"
     echo " -T FILE      Input template to pass to gen_input.py"
     echo " -C FILE      List of cache configurations to test"
-    exit
+    echo "Mode pair options:"
+    echo " -P CPU       CPU to run the 2nd benchmark on"
+    echo " -A           Async. (Don't synchronize the pair at job"
+    echo "              boundries.) Typically used SMT in soft realtime"
 fi
 
 # Name options similarly to rtspin
-while getopts “m:p:l:b:BW:T:C:” opt; do
+while getopts “m:p:l:b:BI:W:T:C:P:A” opt; do
   case $opt in
 # Required
     m) mode=$OPTARG ;;
@@ -27,9 +46,14 @@ while getopts “m:p:l:b:BW:T:C:” opt; do
     b) benchNames=$OPTARG ;;
 # Optional
     B) contend=1 ;;
+    I) iso_mode=$OPTARG ;;
+# DIS
     W) wss_settings=$OPTARG ;;
     T) template_input=$OPTARG ;;
     C) cache_settings=$OPTARG ;;
+# Pair
+    P) core_two=$OPTARG ;;
+    A) async=1 ;;
   esac
 done
 # Reset to read operands
@@ -37,7 +61,7 @@ shift $((OPTIND -1))
 userRunID=$1
 
 # Required argument checking
-if [[ "$mode" != "base" ]] && [[ "$mode" != "dis" ]]; then
+if [[ "$mode" != "base" && "$mode" != "dis" && $mode != pair ]]; then
     echo "Invalid argument: MODE must be either 'base' or 'dis'"
     exit
 fi
@@ -72,14 +96,37 @@ if [[ "$mode" == "dis" ]] && [[ ! -f "$cache_settings" ]]; then
     exit
 fi
 
+# Pair argument checking
+if [[ "$mode" == "pair" && ! -v core_two ]]; then
+    echo "Missing argument: mode=pair reqires -P"
+    exit
+fi
+if [[ "$mode" != "pair" ]] && [[ -v async || -v core_two ]]; then
+    echo "Invalid argument: -A and -P require mode=pair"
+    exit
+fi
+
+# Additional checks
+if [[ $userRunID == "" ]]; then
+    echo "Missing argument: a run ID is required"
+    exit
+fi
+if [[ -v contend && ! -f "/playpen/mc2/imx6q-thrasher/thrasher" ]]; then
+    echo "ERROR: cannot find thrasher binary for -B. Exiting..."
+    exit
+fi
+if [[ $iso_mode == "i" && ! $(uname -a | grep "mc2") ]]; then
+    echo "Isolation mode 'i' requires the MC^2 kernel. Exiting..."
+    exit
+fi
+
 # Check permissions and update state
-if [ "$EUID" -ne 0 ]
-then
+if [[ "$EUID" != 0 ]]; then
     echo "You need to be root to enable interrupt isolation and real-time execution!"
     exit
 fi
 
-echo "Done. Loading benchmark names and input..."
+echo "Loading benchmark names and input..."
 
 # Read the names of each benchmark and load input.
 # This might look a bit scary, but all it does is
@@ -96,11 +143,16 @@ done < $benchNames
 echo "Making sure that binaries are up to date..."
 
 for b in ${bench[@]}; do
-    make bin/$b
-    if [[ $? ]]; then
+    # Build "bin/bench" if bin directory
+    if [[ -d bin ]]; then
+        make bin/$b
+    fi
+    # If that failed, try a different name
+    if [[ "$?" != 0 || ! -d bin ]]; then
         make $b
     fi
-    if [[ ! -f $b && ! -f "./bin/$b" ]]; then
+    # If bench is still not in bin or curr dir, fail
+    if [[ ! -f "./$b" && ! -f "./bin/$b" ]]; then
         echo "Unable to find benchmark $b"
         exit
     fi
@@ -123,26 +175,86 @@ do
     i=$(( $i + 1 ))
 done
 
+echo "Done. Creating cleanup handler..."
+function cleanup {
+    # End contending tasks
+    if [[ -v contend ]]; then
+        killall thrasher
+    fi
+
+    # Restore cache access to everyone
+    echo "L3:0=ffff;1=ffff;2=ffff;3=ffff" > /sys/fs/resctrl/schemata
+    if [[ -e /sys/fs/resctrl/benchmarks ]]; then
+        echo "" > /sys/fs/resctrl/benchmarks/cpus_list
+    fi
+    if [[ -e /sys/fs/resctrl/benchmarks2 ]]; then
+        echo "" > /sys/fs/resctrl/benchmarks2/cpus_list
+    fi
+
+    # Put smp_affinty back the way it was
+    i=0
+    for IRQ in /proc/irq/*
+    do
+        if [ -d $IRQ ]; then
+            echo ${irqList[$i]} 2> /dev/null > $IRQ/smp_affinity_list
+        fi
+        i=$(( $i + 1 ))
+    done
+}
+# This sets the cleanup function to be called when we exit for any reason
+trap cleanup EXIT
+
 echo "Done. Checking for wbinvd module..."
 if [[ ! -f "/proc/wbinvd" ]]; then
     echo "ERROR: wbinvd module not loaded. Exiting..."
     exit
 fi
 
-echo "Done. Setting core $core to 'performance'..."
+echo "Done. Setting cores to 'performance'..."
 echo "performance" > /sys/devices/system/cpu/cpu$core/cpufreq/scaling_governor
+if [[ -v core_two ]]; then
+    echo "performance" > /sys/devices/system/cpu/cpu$core_two/cpufreq/scaling_governor
+fi
 
 # Enable L3 isolation
-echo "Done. Enabling L3 isolation..."
+echo "Done. Setting up L3 isolation..."