From 3b0aa125061b362b23ecf66afe8319e3a268c874 Mon Sep 17 00:00:00 2001 From: Joshua Bakita Date: Thu, 22 Oct 2020 00:08:12 -0400 Subject: Add pair benchmarking and isolation specification into run_bench.sh This allows `run_bench.sh` to replace all previous general paired benchmarking scripts. See `run_everything.sh` for an example of how to use the new script. This also fixes two critical bugs present in the old scripts: 1. The cpus_list was incorrectly set for resctrl when running synchronous pair timing with L2+L3 isolation (i). Each write to that file replaces the previous contents, and it was previously written once for each core. This prevented the entire 1st core from using any L3 cache. 2. The competing task list was indexed improperly in the asynchronous pair timing script. This caused the competing task to always be identical to the task being profiled. These issues combine to imply that in the rejected RTSS'20 paper: - All Level-C SMT timing data is suspect - Level-A and -B SMT timing data with L2+L3 isolation (i) is wildly optimistic --- smt_analysis_rtss20/run_all_pairs_Level-C_DIS.sh | 175 --------------------- smt_analysis_rtss20/run_all_pairs_Level-C_TACLe.sh | 161 ------------------- 2 files changed, 336 deletions(-) delete mode 100755 smt_analysis_rtss20/run_all_pairs_Level-C_DIS.sh delete mode 100755 smt_analysis_rtss20/run_all_pairs_Level-C_TACLe.sh (limited to 'smt_analysis_rtss20') diff --git a/smt_analysis_rtss20/run_all_pairs_Level-C_DIS.sh b/smt_analysis_rtss20/run_all_pairs_Level-C_DIS.sh deleted file mode 100755 index 2ac3fc1..0000000 --- a/smt_analysis_rtss20/run_all_pairs_Level-C_DIS.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash - -firstCore=$1 -secondCore=$2 -maxJobs=$3 -userRunID=$4 -tacleNames=tacleNames.txt - -if [ $# -lt 4 ]; then - echo "Usage $0 [TACLe names file] [--contend]" - exit -fi - -if [ $# -gt 4 ]; then - echo "Using alternate list of TACLe benchmarks from $5" - tacleNames=$5 -fi - -if [ "$EUID" -ne 0 ] -then - echo "You need to be root to enable interrupt isolation and real-time execution!" - exit -fi - -echo "Making sure that binaries are up to date..." -while read i; do - make bin/$i -done < $tacleNames -echo "Done. Disabling real-time throttling..." - -# Turn off rt throttling -echo -1 > /proc/sys/kernel/sched_rt_runtime_us -echo "Done. Redirecting all interrupts to core 0..." - -# Redirect all interrupts to core 0 -i=0 -for IRQ in /proc/irq/* -do - # Skip default_smp_affinity - if [ -d $IRQ ]; then - irqList[$i]=$(cat $IRQ/smp_affinity_list) - echo 0 2> /dev/null > $IRQ/smp_affinity_list - fi - i=$(( $i + 1 )) -done - -# Read the names of each benchmark -j=0 -while read i; do - tacleProg[$j]=$i - j=$(( $j + 1 )) -done < $tacleNames - -echo "Done. Checking for wbinvd module..." -if [[ ! -f "/proc/wbinvd" ]]; then - echo "ERROR: wbinvd module not loaded. Exiting..." - exit -fi - -# Enable L3 isolation -echo "Done. Enabling L3 isolation..." -mount -t resctrl resctrl /sys/fs/resctrl -mkdir -p /sys/fs/resctrl/benchmarks -echo $firstCore > /sys/fs/resctrl/benchmarks/cpus_list -echo $secondCore > /sys/fs/resctrl/benchmarks/cpus_list -# Reset global bandwith control and remove L3 from global -echo "L3:0=ffff;1=ffff;2=ffff;3=0000" > /sys/fs/resctrl/schemata -echo "MB:0=2048;1=2048;2=2048;3=2048" > /sys/fs/resctrl/schemata -# Alloc L3 to benchmark -echo "L3:0=0000;1=0000;2=0000;3=ffff" > /sys/fs/resctrl/benchmarks/schemata -echo "MB:0=2048;1=2048;2=2048;3=2048" > /sys/fs/resctrl/benchmarks/schemata -echo "Done. Verifying configuration with user..." - -# Generate file name string -# We append to this as we parse the environment settings -runID=$(date +"%b%d-%H") - -# Confirm configuration with user -echo "=== Global Config ===" -cat /sys/fs/resctrl/schemata -echo "=== Core $firstCore and $secondCore Config ===" -cat /sys/fs/resctrl/benchmarks/schemata -if [[ $6 == "--contend" ]]; then - if [[ ! -f "/playpen/mc2/imx6q-thrasher/thrasher" ]]; then - echo "ERROR: thrasher binary not fonud. Exiting..." - exit - fi - echo "Will run 6 contending tasks" - runID=$runID-c -else - runID=$runID-xc -fi -if uname -a | grep -q "mc2"; then - echo "MC^2 Autodetected. Cache coloring will be DISABLED." - runID=$runID-i -else - echo "MC^2 not detected. Cache coloring will be DISABLED." - runID=$runID-xi -fi -echo "Results will be saved as $runID-$userRunID-LC-DIS.txt" -echo "Press enter to confirm environment, Ctrl-C to exit..." -read - -# Start contending tasks -if [[ $6 == "--contend" ]]; then - echo "Done. Starting 6 contending tasks..." - # Run two contending tasks on each other CCX - taskset -c 1 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 2 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 5 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 6 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 9 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 10 /playpen/mc2/imx6q-thrasher/thrasher & -fi -sleep 1 # Wait for contending tasks to start -echo "Done. Beginning benchmarks..." - -num_tests=$(wc -l < $tacleNames) -for (( i = 0; i < $num_tests ; i++ )) -do - # For level-C analysis, we must run all vs all - for (( j = 0; j < $num_tests ; j++ )) #loop through programs - do - # Autodetect MC^2 - # Get DIS to loop continuously in the background by infinite restarts - if uname -a | grep -q "mc2"; then - bash -c "while true; do chrt -r 97 numactl --interleave=all taskset -c $firstCore ./bin/${tacleProg[$i]} NULL -1 $firstCore NULL 0 &> /dev/null; done" & PID1=$!; - else - bash -c "while true; do chrt -r 97 taskset -c $firstCore ./bin/${tacleProg[$i]} NULL -1 $firstCore NULL 0 &> /dev/null; done" & PID1=$!; - fi - # Each sample runs in its own process - for ((k=0;k<=maxJobs;k++)); do - if uname -a | grep -q "mc2"; then - chrt -r 97 numactl --interleave=all taskset -c $secondCore ./bin/${tacleProg[$i]} ${tacleProg[$i]}"+"${tacleProg[$j]} $maxJobs $secondCore $runID-$userRunID-LC-DIS 1 & PID2=$!; - else - chrt -r 97 taskset -c $secondCore ./bin/${tacleProg[$i]} ${tacleProg[$i]}"+"${tacleProg[$j]} $maxJobs $secondCore $runID-$userRunID-LC-DIS 1 & PID2=$!; - fi - done - # The paired task runs continuously in this configuration, so we have - # to kill it once the task that we're measuring finishes. - wait $PID2 - kill $PID1 - # Killing bash will not kill its children by default - work aronud that here - killall transitive &> /dev/null - killall matrix &> /dev/null - killall pointer &> /dev/null - killall field &> /dev/null - killall neighborhood &> /dev/null - killall update &> /dev/null - killall gen_input.py &> /dev/null - echo ${tacleProg[$i]} ${tacleProg[$j]} - done - echo COMPLETE: ${tacleProg[$i]} -done - -# End contending tasks -if [[ $6 == "--contend" ]]; then - killall thrasher -fi - -# Remove semaphores from system -# Leaving them won't hurt these tests, but would be messy and is bad practice -# TODO: Do this directly in the benchmarks. They should clean up after themselves -./bin/cleanupSemaphores - -# Put smp_affinty back the way it was -i=0 -for IRQ in /proc/irq/* -do - if [ -d $IRQ ]; then - echo ${irqList[$i]} 2> /dev/null > $IRQ/smp_affinity_list - fi - i=$(( $i + 1 )) -done - diff --git a/smt_analysis_rtss20/run_all_pairs_Level-C_TACLe.sh b/smt_analysis_rtss20/run_all_pairs_Level-C_TACLe.sh deleted file mode 100755 index 923dba0..0000000 --- a/smt_analysis_rtss20/run_all_pairs_Level-C_TACLe.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash - -firstCore=$1 -secondCore=$2 -maxJobs=$3 -userRunID=$4 -tacleNames=tacleNames.txt - -if [ $# -lt 4 ]; then - echo "Usage $0 [TACLe names file] [--contend]" - exit -fi - -if [ $# -gt 4 ]; then - echo "Using alternate list of TACLe benchmarks from $5" - tacleNames=$5 -fi - -if [ "$EUID" -ne 0 ] -then - echo "You need to be root to enable interrupt isolation and real-time execution!" - exit -fi - -echo "Making sure that binaries are up to date..." -while read i; do - make bin/$i -done < $tacleNames -echo "Done. Disabling real-time throttling..." - -# Turn off rt throttling -echo -1 > /proc/sys/kernel/sched_rt_runtime_us -echo "Done. Redirecting all interrupts to core 0..." - -# Redirect all interrupts to core 0 -i=0 -for IRQ in /proc/irq/* -do - # Skip default_smp_affinity - if [ -d $IRQ ]; then - irqList[$i]=$(cat $IRQ/smp_affinity_list) - echo 0 2> /dev/null > $IRQ/smp_affinity_list - fi - i=$(( $i + 1 )) -done - -# Read the names of each benchmark -j=0 -while read i; do - tacleProg[$j]=$i - j=$(( $j + 1 )) -done < $tacleNames - -echo "Done. Checking for wbinvd module..." -if [[ ! -f "/proc/wbinvd" ]]; then - echo "ERROR: wbinvd module not loaded. Exiting..." - exit -fi - -# Enable L3 isolation -echo "Done. Enabling L3 isolation..." -mount -t resctrl resctrl /sys/fs/resctrl -mkdir -p /sys/fs/resctrl/benchmarks -echo $firstCore > /sys/fs/resctrl/benchmarks/cpus_list -echo $secondCore > /sys/fs/resctrl/benchmarks/cpus_list -# Reset global bandwith control and remove L3 from global -echo "L3:0=ffff;1=ffff;2=ffff;3=0000" > /sys/fs/resctrl/schemata -echo "MB:0=2048;1=2048;2=2048;3=2048" > /sys/fs/resctrl/schemata -# Alloc L3 to benchmark -echo "L3:0=0000;1=0000;2=0000;3=ffff" > /sys/fs/resctrl/benchmarks/schemata -echo "MB:0=2048;1=2048;2=2048;3=2048" > /sys/fs/resctrl/benchmarks/schemata -echo "Done. Verifying configuration with user..." - -# Generate file name string -# We append to this as we parse the environment settings -runID=$(date +"%b%d-%H") - -# Confirm configuration with user -echo "=== Global Config ===" -cat /sys/fs/resctrl/schemata -echo "=== Core $firstCore and $secondCore Config ===" -cat /sys/fs/resctrl/benchmarks/schemata -if [[ $6 == "--contend" ]]; then - if [[ ! -f "/playpen/mc2/imx6q-thrasher/thrasher" ]]; then - echo "ERROR: thrasher binary not fonud. Exiting..." - exit - fi - echo "Will run 6 contending tasks" - runID=$runID-c -else - runID=$runID-xc -fi -if uname -a | grep -q "mc2"; then - echo "MC^2 Autodetected. Cache coloring will be DISABLED." - runID=$runID-i -else - echo "MC^2 not detected. Cache coloring will be DISABLED." - runID=$runID-xi -fi -echo "Results will be saved as $runID-$userRunID-LC-TACLe.txt" -echo "Press enter to confirm environment, Ctrl-C to exit..." -read - -# Start contending tasks -if [[ $6 == "--contend" ]]; then - echo "Done. Starting 6 contending tasks..." - # Run two contending tasks on each other CCX - taskset -c 1 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 2 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 5 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 6 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 9 /playpen/mc2/imx6q-thrasher/thrasher & - taskset -c 10 /playpen/mc2/imx6q-thrasher/thrasher & -fi -sleep 1 # Wait for contending tasks to start -echo "Done. Beginning benchmarks..." - -num_tests=$(wc -l < $tacleNames) -for (( i = 0; i < $num_tests ; i++ )) -do - # For level-C analysis, we must run all vs all - for (( j = 0; j < $num_tests ; j++ )) #loop through programs - do - # Autodetect MC^2 - if uname -a | grep -q "mc2"; then - # -1 loops means loop forever - chrt -r 97 numactl --interleave=all taskset -c $firstCore ./bin/${tacleProg[$i]} NULL -1 $firstCore NULL 0 & PID1=$!; - chrt -r 97 numactl --interleave=all taskset -c $secondCore ./bin/${tacleProg[$i]} ${tacleProg[$i]}"+"${tacleProg[$j]} $maxJobs $secondCore $runID-$userRunID-LC-TACLe 1 & PID2=$!; - else - chrt -r 97 taskset -c $firstCore ./bin/${tacleProg[$i]} NULL -1 $firstCore NULL 0 & PID1=$!; - chrt -r 97 taskset -c $secondCore ./bin/${tacleProg[$i]} ${tacleProg[$i]}"+"${tacleProg[$j]} $maxJobs $secondCore $runID-$userRunID-LC-TACLe 1 & PID2=$!; - fi - # The paired task runs continuously in this configuration, so we have - # to kill it once the task that we're measuring finishes. - wait $PID2 - kill $PID1 - echo ${tacleProg[$i]} ${tacleProg[$j]} - done - echo COMPLETE: ${tacleProg[$i]} -done - -# End contending tasks -if [[ $6 == "--contend" ]]; then - killall thrasher -fi - -# Remove semaphores from system -# Leaving them won't hurt these tests, but would be messy and is bad practice -# TODO: Do this directly in the benchmarks. They should clean up after themselves -./bin/cleanupSemaphores - -# Put smp_affinty back the way it was -i=0 -for IRQ in /proc/irq/* -do - if [ -d $IRQ ]; then - echo ${irqList[$i]} 2> /dev/null > $IRQ/smp_affinity_list - fi - i=$(( $i + 1 )) -done - -- cgit v1.2.2