Unify all the versions of extra.h into a single multipurpose header

There was previously a huge amount of shared code that had to be
copied back and forth. This should reduce the maintenance burden
by containing all future changes to a single file.

New unified library is fully backwards-compatible but also
introduces and the easy-to-use `for_each_job` macro which
replaces the specific `for(...) START_LOOP ... STOP_LOOP`
format requirement and is generally much harder to abuse.

New unified library also automatically cleans up its shared memory
and semaphores, so this commit also removes the separate
`cleanupSemaphores` binary.

I also found a precursor of `extra.h` written by Sims in
`litmusStuff.h`. This code is only interesting for historical
purposes, so it is also removed in this commit.

This commit also adds debug options to all the Makefiles and
silences rm's complaints about non-existent files in make clean.

54 files changed, 317 insertions, 446 deletions

diff --git a/all_pairs/Makefile b/all_pairs/Makefile
index 36cf55a..0042372 100644
--- a/all_pairs/Makefile
+++ b/all_pairs/Makefile
@@ -1,57 +1,73 @@
-CC = gcc
-CFLAGS = -pthread -O2
+LIBLITMUS ?= /media/speedy/litmus/liblitmus
+CC ?= gcc
+CFLAGS = -pthread -O2 -I.. -DPAIRED
 LDFLAGS = -lrt
-all: bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec bin/cleanupSemaphores
+COMMON = ../extra.h
+
+# Handle cases where we're also profiling with the MMDC on the i.MX6Q
+ifneq ($(shell grep "define MMDC 1" ../extra.h),)
+        COMMON += /media/speedy/litmus/tools/mmdc/mmdc.c
+endif
+
+# Include all the LITMUS^RT headers if we're using it
+ifneq ($(shell grep "define LITMUS 1" ../extra.h),)
+        CFLAGS += -I${LIBLITMUS}/include -I${LIBLITMUS}/arch/arm/include
+        LDFLAGS += -L${LIBLITMUS} -llitmus
+endif
+
+all: bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec
+
+debug: all
+debug: CFLAGS += -ggdb3
 
 .PHONY: clean
 clean:
-        rm bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec bin/cleanupSemaphores
+        rm -f bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec
 
-bin/cleanupSemaphores: ./source/extra.h  ./source/cleanupSemaphores.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/cjpeg_wrbmp: ./source/extra.h  ./source/cjpeg_wrbmp/cjpeg_wrbmp.c ./source/cjpeg_wrbmp/input.c
+bin/cjpeg_wrbmp: ${COMMON} ./source/cjpeg_wrbmp/cjpeg_wrbmp.c ./source/cjpeg_wrbmp/input.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/huff_enc: ./source/extra.h  ./source/huff_enc/huff_enc.c
+bin/huff_enc: ${COMMON} ./source/huff_enc/huff_enc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/gsm_enc: ./source/extra.h  ./source/gsm_enc/gsm_enc.c
+bin/gsm_enc: ${COMMON} ./source/gsm_enc/gsm_enc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/dijkstra: ./source/extra.h  ./source/dijkstra/dijkstra.c ./source/dijkstra/input.c
+bin/dijkstra: ${COMMON} ./source/dijkstra/dijkstra.c ./source/dijkstra/input.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/h264_dec: ./source/extra.h  ./source/h264_dec/h264_dec.c ./source/h264_dec/h264_decinput.c
+bin/h264_dec: ${COMMON} ./source/h264_dec/h264_dec.c ./source/h264_dec/h264_decinput.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/susan: ./source/extra.h  ./source/susan/input.c ./source/susan/susan.c ./source/susan/wccfile.c ./source/susan/wcclibm.c ./source/susan/wccmalloc.c
+bin/susan: ${COMMON} ./source/susan/input.c ./source/susan/susan.c ./source/susan/wccfile.c ./source/susan/wcclibm.c ./source/susan/wccmalloc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/adpcm_enc: ./source/extra.h  ./source/adpcm_enc/adpcm_enc.c
+bin/adpcm_enc: ${COMMON} ./source/adpcm_enc/adpcm_enc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/rijndael_dec: ./source/extra.h  ./source/rijndael_dec/aes.c ./source/rijndael_dec/input_small_enc.c ./source/rijndael_dec/rijndael_dec.c ./source/rijndael_dec/rijndael_dec_libc.c
+bin/rijndael_dec: ${COMMON} ./source/rijndael_dec/aes.c ./source/rijndael_dec/input_small_enc.c ./source/rijndael_dec/rijndael_dec.c ./source/rijndael_dec/rijndael_dec_libc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/huff_dec: ./source/extra.h  ./source/huff_dec/huff_dec.c
+bin/huff_dec: ${COMMON} ./source/huff_dec/huff_dec.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/rijndael_enc: ./source/extra.h  ./source/rijndael_enc/aes.c ./source/rijndael_enc/input_small.c ./source/rijndael_enc/rijndael_enc.c ./source/rijndael_enc/rijndael_enc_libc.c
+bin/rijndael_enc: ${COMMON} ./source/rijndael_enc/aes.c ./source/rijndael_enc/input_small.c ./source/rijndael_enc/rijndael_enc.c ./source/rijndael_enc/rijndael_enc_libc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/gsm_dec: ./source/extra.h  ./source/gsm_dec/gsm_dec.c
+bin/gsm_dec: ${COMMON} ./source/gsm_dec/gsm_dec.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/anagram: ./source/extra.h  ./source/anagram/anagram.c ./source/anagram/anagram_input.c ./source/anagram/anagram_stdlib.c
+bin/anagram: ${COMMON} ./source/anagram/anagram.c ./source/anagram/anagram_input.c ./source/anagram/anagram_stdlib.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/epic: ./source/extra.h  ./source/epic/epic.c
+bin/epic: ${COMMON} ./source/epic/epic.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/ammunition: ./source/extra.h  ./source/ammunition/ammunition.c ./source/ammunition/ammunition_libc.c ./source/ammunition/arithm.c ./source/ammunition/bits.c
+bin/ammunition: ${COMMON} ./source/ammunition/ammunition.c ./source/ammunition/ammunition_libc.c ./source/ammunition/arithm.c ./source/ammunition/bits.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/g723_enc: ./source/extra.h  ./source/g723_enc/g723_enc.c
+bin/g723_enc: ${COMMON} ./source/g723_enc/g723_enc.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/ndes: ./source/extra.h  ./source/ndes/ndes.c
+bin/ndes: ${COMMON} ./source/ndes/ndes.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/petrinet: ./source/extra.h  ./source/petrinet/petrinet.c
+bin/petrinet: ${COMMON} ./source/petrinet/petrinet.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/statemate: ./source/extra.h  ./source/statemate/statemate.c
+bin/statemate: ${COMMON} ./source/statemate/statemate.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/cjpeg_transupp: ./source/extra.h  ./source/cjpeg_transupp/cjpeg_transupp.c
+bin/cjpeg_transupp: ${COMMON} ./source/cjpeg_transupp/cjpeg_transupp.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/mpeg2: ./source/extra.h  ./source/mpeg2/mpeg2.c
+bin/mpeg2: ${COMMON} ./source/mpeg2/mpeg2.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/fmref: ./source/extra.h  ./source/fmref/fmref.c ./source/fmref/wcclibm.c
+bin/fmref: ${COMMON} ./source/fmref/fmref.c ./source/fmref/wcclibm.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/audiobeam: ./source/extra.h  ./source/audiobeam/audiobeam.c ./source/audiobeam/audiobeaminput.c ./source/audiobeam/audiobeamlibmalloc.c ./source/audiobeam/audiobeamlibm.c
+bin/audiobeam: ${COMMON} ./source/audiobeam/audiobeam.c ./source/audiobeam/audiobeaminput.c ./source/audiobeam/audiobeamlibmalloc.c ./source/audiobeam/audiobeamlibm.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
-bin/adpcm_dec: ./source/extra.h  ./source/adpcm_dec/adpcm_dec.c
+bin/adpcm_dec: ${COMMON} ./source/adpcm_dec/adpcm_dec.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+
diff --git a/all_pairs/source/adpcm_dec/adpcm_dec.c b/all_pairs/source/adpcm_dec/adpcm_dec.c
index 04a5746..368e98d 100644
--- a/all_pairs/source/adpcm_dec/adpcm_dec.c
+++ b/all_pairs/source/adpcm_dec/adpcm_dec.c
@@ -35,7 +35,7 @@
   Forward declaration of functions
 */
 
-#include "../extra.h"
+#include "extra.h"
 
 void adpcm_dec_decode( int );
 int adpcm_dec_filtez( int *bpl, int *dlt );
diff --git a/all_pairs/source/adpcm_enc/adpcm_enc.c b/all_pairs/source/adpcm_enc/adpcm_enc.c
index 464768f..777aaf5 100644
--- a/all_pairs/source/adpcm_enc/adpcm_enc.c
+++ b/all_pairs/source/adpcm_enc/adpcm_enc.c
@@ -29,7 +29,7 @@
 
 /* common sampling rate for sound cards on IBM/PC */
 
-#include "../extra.h"
+#include "extra.h"
 #define SAMPLE_RATE 11025
 
 #define PI 3141
diff --git a/all_pairs/source/ammunition/ammunition.c b/all_pairs/source/ammunition/ammunition.c
index 269f4c0..052520e 100644
--- a/all_pairs/source/ammunition/ammunition.c
+++ b/all_pairs/source/ammunition/ammunition.c
@@ -24,7 +24,7 @@
 
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "bits.h"
 #include "arithm.h"
 #include "ammunition_stdlib.h"
diff --git a/all_pairs/source/anagram/anagram.c b/all_pairs/source/anagram/anagram.c
index b458fd2..5c1f29a 100644
--- a/all_pairs/source/anagram/anagram.c
+++ b/all_pairs/source/anagram/anagram.c
@@ -157,7 +157,7 @@
    steps to FindAnagram.
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "anagram_ctype.h"
 #include "anagram_stdlib.h"
 #include "anagram_strings.h"
diff --git a/all_pairs/source/audiobeam/audiobeam.c b/all_pairs/source/audiobeam/audiobeam.c
index 208de80..50ebfff 100644
--- a/all_pairs/source/audiobeam/audiobeam.c
+++ b/all_pairs/source/audiobeam/audiobeam.c
@@ -23,7 +23,7 @@
   Include section
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "audiobeamlibm.h"
 #include "audiobeamlibmalloc.h"
 #include "audiobeam.h"
diff --git a/all_pairs/source/cjpeg_transupp/cjpeg_transupp.c b/all_pairs/source/cjpeg_transupp/cjpeg_transupp.c
index 3f48539..5ec7e5e 100644
--- a/all_pairs/source/cjpeg_transupp/cjpeg_transupp.c
+++ b/all_pairs/source/cjpeg_transupp/cjpeg_transupp.c
@@ -29,7 +29,7 @@
   Include section
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "jpeglib.h"
 
 
diff --git a/all_pairs/source/cjpeg_wrbmp/cjpeg_wrbmp.c b/all_pairs/source/cjpeg_wrbmp/cjpeg_wrbmp.c
index 278725c..3c8d7ec 100644
--- a/all_pairs/source/cjpeg_wrbmp/cjpeg_wrbmp.c
+++ b/all_pairs/source/cjpeg_wrbmp/cjpeg_wrbmp.c
@@ -24,7 +24,7 @@
 
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "cdjpeg.h"
 
 #ifdef CJPEG_WRBMP_BMP_SUPPORTED
diff --git a/all_pairs/source/cleanupSemaphores.c b/all_pairs/source/cleanupSemaphores.c
deleted file mode 100644
index dde2bb3..0000000
--- a/all_pairs/source/cleanupSemaphores.c
+++ /dev/null
@@ -1,6 +0,0 @@
-#include <semaphore.h>
-
-int main(){
-        sem_unlink("/firstTacleSem");
-        sem_unlink("/secondTacleSem");
-}
diff --git a/all_pairs/source/dijkstra/dijkstra.c b/all_pairs/source/dijkstra/dijkstra.c
index 1b6a52f..333fd43 100644
--- a/all_pairs/source/dijkstra/dijkstra.c
+++ b/all_pairs/source/dijkstra/dijkstra.c
@@ -17,7 +17,7 @@
 
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "input.h"
 
 /*
diff --git a/all_pairs/source/empty.c b/all_pairs/source/empty.c
index c7c5f5d..6681ba7 100644
--- a/all_pairs/source/empty.c
+++ b/all_pairs/source/empty.c
@@ -1,4 +1,4 @@
-#include "../extra.h"
+#include "extra.h"
 
 int main(int argc, char **argv)
 {
diff --git a/all_pairs/source/epic/epic.c b/all_pairs/source/epic/epic.c
index e258a4a..a1e344c 100644
--- a/all_pairs/source/epic/epic.c
+++ b/all_pairs/source/epic/epic.c
@@ -35,7 +35,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 #include "epic.h"
 
 #define X_SIZE  64
diff --git a/all_pairs/source/extra.h b/all_pairs/source/extra.h
deleted file mode 100644
index 0681250..0000000
--- a/all_pairs/source/extra.h
+++ /dev/null
@@ -1,211 +0,0 @@
-/**
- * Copyright 2019 Sims Hill Osborne and Joshua Bakita
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- **/
-#define _GNU_SOURCE
-#include <fcntl.h>
-#include <limits.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <time.h>
-#include <unistd.h>
-#include <sched.h>
-
-// This is only visible if _GNU_SOURCE is defined, and that define does not
-// come along to places where this file is included. Address this by manually
-// forcing it into the global namespace.
-extern int sched_getcpu();
-
-// Benchmarks use SET_UP, START_LOOP, STOP_LOOP, and WRITE_TO_FILE
-// These are macros so that we can declare and maintain additional state inside
-// the benchmark.
-#define SET_UP if (argc != 8) {\
-        printf("Usage: %s <name> <runs> <my core> <other core> <other program> <runID> <lockID>", argv[0]);\
-        exit(1);\
-    }\
-    char * thisProgram = argv[1];\
-    int maxJobs = atoi(argv[2]);\
-    unsigned int thisCore = atoi(argv[3]);\
-    unsigned int otherCore = atoi(argv[4]);\
-    thisCore = sched_getcpu();\
-    char * otherProgram = argv[5];\
-    char * runID = argv[6];\
-    int lockID = atoi(argv[7]);\
-    struct timespec _start, _end;\
-    int jobsComplete;\
-    int jobs_complete = -1;\
-    long * startS = malloc(sizeof(long) *maxJobs);\
-    long * startN = malloc(sizeof(long) *maxJobs);\
-    long * endS = malloc(sizeof(long) *maxJobs);\
-    long * endN = malloc(sizeof(long) *maxJobs);\
-    char * bigArray;\
-    char fileName[strlen(runID) + 5];\
-    strcpy(fileName, runID);\
-    strcat(fileName, ".txt");\
-    mlockall(MCL_CURRENT || MCL_FUTURE);\
-    sem_t *firstSem=sem_open("/firstTacleSem", O_CREAT, 644, 0);\
-    if (firstSem == SEM_FAILED) {\
-        perror("Error opening/creating first semaphore");\
-        exit(1);\
-    }\
-    sem_t *secondSem=sem_open("/secondTacleSem", O_CREAT, 644, 0);\
-    if (secondSem == SEM_FAILED) {\
-        perror("Error opening/creating second semaphore");\
-        exit(1);\
-    }\
-    int barrier_file = shm_open("/TacleBarrier", O_CREAT | O_RDWR, 644);\
-    if (barrier_file == -1) {\
-        perror("Error creating shared memory");\
-        exit(1);\
-    }\
-    /* This sets our shared file to be one byte of '\0'*/ \
-    if (ftruncate(barrier_file, 1) == -1) {\
-        perror("Error setting size of shared memory");\
-        exit(1);\
-    }\
-    char * barrier = mmap(NULL, 1, PROT_WRITE, MAP_SHARED, barrier_file, 0);\
-    if (barrier == MAP_FAILED) {\
-        perror("Error mapping shared memory");\
-        exit(1);\
-    }\
-    int val;
-
-#define SAVE_RESULTS \
-     if(jobs_complete >= maxJobs) {\
-        fprintf(stderr, "Max jobs setting too small! Trying to record job #%d when we only have space for %d jobs. Exiting...\n", jobs_complete, maxJobs);\
-        exit(1);\
-    }\
-    if (jobs_complete > -1){\
-        startS[jobs_complete]=_start.tv_sec;\
-        startN[jobs_complete]=_start.tv_nsec;\
-        endS[jobs_complete]=_end.tv_sec;\
-        endN[jobs_complete]=_end.tv_nsec;\
-    }
-
-#define WRITE_TO_FILE {\
-    munlockall();\
-    FILE *fp = fopen(fileName, "a");\
-    if (fp == NULL) {\
-        perror("Error opening file. \n");\
-        exit(1);\
-    }\
-    for(int i = 0; i <= jobs_complete; i++){\
-        fprintf(fp, "%s %s %u %u %d %ld %ld %ld %ld %s %d \n",\
-        thisProgram, otherProgram, thisCore, otherCore, maxJobs,\
-        startS[i], startN[i], endS[i], endN[i],\
-        runID, i);\
-    }\
-    fclose(fp);\
-    /* Clean up the barrier synchronization shared memory */\
-    munmap(barrier, 1);\
-    shm_unlink("/TacleBarrier");\
-    free(startS);\
-    free(startN);\
-    free(endS);\
-    free(endN);\
-}
-
-// Call the wbinvld instruction (it's in a kernel module due to it being ring-0)
-#define FLUSH_CACHES FILE *fp = fopen("/proc/wbinvd", "r");\
-    if (fp == NULL) {\
-        perror("Cache flush module interface cannot be opened");\
-        exit(1);\
-    }\
-    char dummy;\
-    if (fread(&dummy, 1, 1, fp) == 0) {\
-        perror("Unable to access cache flush module interface");\
-        exit(1);\
-    }\
-    fclose(fp);
-
-// These timers should just be aliases to the hardware counters w/ some small adjustments
-#define START_TIMER clock_gettime(CLOCK_MONOTONIC, &_start);
-#define STOP_TIMER clock_gettime(CLOCK_MONOTONIC, &_end);
-
-//check value of sem
-//if sem=0, unlock
-//if sem=1, spin
-
-#define SLEEP nanosleep((const struct timespec[]){{0, 1000000}}, NULL);
-
-#define FIRST_UNLOCK if (lockID == 1) {\
-        if (sem_post(secondSem) != 0) {\
-            perror("Unable to unlock second semaphore");\
-            exit(1);\
-        }\
-    } \
-    else {\
-        if (sem_post(firstSem) != 0) {\
-            perror("Unable to unlock first semaphore");\
-            exit(1);\
-        }\
-    } \
-
-#define FIRST_LOCK if (lockID == 1) {\
-        if (sem_wait(firstSem) != 0) {\
-            perror("Unable to wait on first semaphore");\
-            exit(1);\
-        }\
-    }\
-    else {\
-        if (sem_wait(secondSem) != 0) {\
-            perror("Unable to wait on second semaphore");\
-            exit(1);\
-        }\
-    }
-
-
-#define SECOND_UNLOCK   if (lockID==1){sem_post(fourthSem) ; }\
-            else {sem_post(thirdSem) ; }
-
-#define SECOND_LOCK if (lockID==1){sem_wait(thirdSem); }\
-            else {sem_wait(fourthSem); }
-
-#define BARRIER_SYNC if (__sync_bool_compare_and_swap(barrier, 0, 1)) {\
-        while (!__sync_bool_compare_and_swap(barrier, 0, 0)) {};\
-    }\
-    else {\
-        __sync_bool_compare_and_swap(barrier, 1, 0);\
-    }
-
-#define START_LOOP FIRST_UNLOCK FIRST_LOCK sched_yield(); FLUSH_CACHES BARRIER_SYNC START_TIMER
-#define STOP_LOOP STOP_TIMER jobs_complete++; SAVE_RESULTS
-
-
-/*
-Intended structure
-
-main
-SET_UP
-notice that STOP LOOP negates the ++ if outout=0
-for (jobsComplete=0; jobsComplete<maxJobs; jobsComplete++){
-        START_LOOP
-        tacleInit();
-        tacleMain();
-        STOP_LOOP
-}
-WRITE_TO_FILE
-tacleReturn 
-*/
diff --git a/all_pairs/source/fmref/fmref.c b/all_pairs/source/fmref/fmref.c
index bcc7ce6..e4c9aaa 100644
--- a/all_pairs/source/fmref/fmref.c
+++ b/all_pairs/source/fmref/fmref.c
@@ -5,7 +5,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 #include "wcclibm.h"
 #ifndef M_PI
 #define M_PI        3.1415926535897932384626433832795
diff --git a/all_pairs/source/g723_enc/g723_enc.c b/all_pairs/source/g723_enc/g723_enc.c
index 331c821..209e3ce 100644
--- a/all_pairs/source/g723_enc/g723_enc.c
+++ b/all_pairs/source/g723_enc/g723_enc.c
@@ -31,7 +31,7 @@
    included in this Recommendation.
 */
 
-#include "../extra.h"
+#include "extra.h"
 struct g723_enc_state_t {
   long yl;  /* Locked or steady state step size multiplier. */
   short yu; /* Unlocked or non-steady state step size multiplier. */
diff --git a/all_pairs/source/gsm_dec/gsm_dec.c b/all_pairs/source/gsm_dec/gsm_dec.c
index 06d24d5..ba9a9e9 100644
--- a/all_pairs/source/gsm_dec/gsm_dec.c
+++ b/all_pairs/source/gsm_dec/gsm_dec.c
@@ -21,7 +21,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 #include "gsm.h"
 #include "add.h"
 #include "data.h"
diff --git a/all_pairs/source/gsm_enc/gsm_enc.c b/all_pairs/source/gsm_enc/gsm_enc.c
index 3f051ec..e8154bb 100644
--- a/all_pairs/source/gsm_enc/gsm_enc.c
+++ b/all_pairs/source/gsm_enc/gsm_enc.c
@@ -5,7 +5,7 @@
  * details.  THERE IS ABSOLUTELY NO WARRANTY FOR THIS SOFTWARE.
  */
 
-#include "../extra.h"
+#include "extra.h"
 #include "private.h"
 
 /*
diff --git a/all_pairs/source/h264_dec/h264_dec.c b/all_pairs/source/h264_dec/h264_dec.c
index 1fed68b..e74bec9 100644
--- a/all_pairs/source/h264_dec/h264_dec.c
+++ b/all_pairs/source/h264_dec/h264_dec.c
@@ -25,7 +25,7 @@
   Include section
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "h264_dec.h"
 
 
diff --git a/all_pairs/source/huff_dec/huff_dec.c b/all_pairs/source/huff_dec/huff_dec.c
index b83b99d..58b40d3 100644
--- a/all_pairs/source/huff_dec/huff_dec.c
+++ b/all_pairs/source/huff_dec/huff_dec.c
@@ -49,7 +49,7 @@ new schemes (this item is not a must)
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 typedef struct s_tree {
   unsigned int byte; /* A byte has to be coded as an unsigned integer to
                         allow a node to have a value over 255 */
diff --git a/all_pairs/source/huff_enc/huff_enc.c b/all_pairs/source/huff_enc/huff_enc.c
index 133fe95..0d74916 100644
--- a/all_pairs/source/huff_enc/huff_enc.c
+++ b/all_pairs/source/huff_enc/huff_enc.c
@@ -50,7 +50,7 @@ new schemes (this item is not a must)
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 typedef struct huff_enc_s_tree {
   unsigned int byte; /* A byte has to be coded as an unsigned integer to
                         allow a node to have a value over 255 */
diff --git a/all_pairs/source/mpeg2/mpeg2.c b/all_pairs/source/mpeg2/mpeg2.c
index a810744..0567225 100644
--- a/all_pairs/source/mpeg2/mpeg2.c
+++ b/all_pairs/source/mpeg2/mpeg2.c
@@ -48,7 +48,7 @@
 /*
   Forward declaration of data types
 */
-#include "../extra.h"
+#include "extra.h"
 struct mbinfo;
 
 
diff --git a/all_pairs/source/ndes/ndes.c b/all_pairs/source/ndes/ndes.c
index b0f494c..f169c62 100644
--- a/all_pairs/source/ndes/ndes.c
+++ b/all_pairs/source/ndes/ndes.c
@@ -17,7 +17,7 @@
 */
 
 /* A read from this address will result in an known value of 1 */
-#include "../extra.h"
+#include "extra.h"
 #define KNOWN_VALUE 1
 #define NDES_WORSTCASE 1
 /*
diff --git a/all_pairs/source/petrinet/petrinet.c b/all_pairs/source/petrinet/petrinet.c
index cabb40d..6ef3392 100644
--- a/all_pairs/source/petrinet/petrinet.c
+++ b/all_pairs/source/petrinet/petrinet.c
@@ -25,7 +25,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 
 #ifdef PROFILING
 #include <stdio.h>
diff --git a/all_pairs/source/rijndael_dec/rijndael_dec.c b/all_pairs/source/rijndael_dec/rijndael_dec.c
index 93bb424..61db767 100644
--- a/all_pairs/source/rijndael_dec/rijndael_dec.c
+++ b/all_pairs/source/rijndael_dec/rijndael_dec.c
@@ -38,7 +38,7 @@
   -----------------------------------------------------------------------
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "aes.h"
 #include "rijndael_dec_libc.h"
 
diff --git a/all_pairs/source/rijndael_enc/rijndael_enc.c b/all_pairs/source/rijndael_enc/rijndael_enc.c
index 0c10353..6c85eee 100644
--- a/all_pairs/source/rijndael_enc/rijndael_enc.c
+++ b/all_pairs/source/rijndael_enc/rijndael_enc.c
@@ -38,7 +38,7 @@
   -----------------------------------------------------------------------
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "aes.h"
 #include "rijndael_enc_libc.h"
 
diff --git a/all_pairs/source/statemate/statemate.c b/all_pairs/source/statemate/statemate.c
index 00491e5..888d0fa 100644
--- a/all_pairs/source/statemate/statemate.c
+++ b/all_pairs/source/statemate/statemate.c
@@ -27,7 +27,7 @@
   Macro definitions
 */
 
-#include "../extra.h"
+#include "extra.h"
 
 #define SYS_bit_get(a,b) (a)[(b)]
 #define SYS_bit_clr(a,b) (a)[(b)] = 0
diff --git a/all_pairs/source/susan/susan.c b/all_pairs/source/susan/susan.c
index 4bc4cb8..dc27fb1 100644
--- a/all_pairs/source/susan/susan.c
+++ b/all_pairs/source/susan/susan.c
@@ -269,7 +269,7 @@
 
 \**********************************************************************/
 
-#include "../extra.h"
+#include "extra.h"
 #include "wcclibm.h"
 #include "wccfile.h"
 #include "wccmalloc.h"
diff --git a/baseline/Makefile b/baseline/Makefile
index ef39b7d..40b57f4 100644
--- a/baseline/Makefile
+++ b/baseline/Makefile
@@ -1,25 +1,28 @@
 LIBLITMUS ?= /media/speedy/litmus/liblitmus
 CC ?= gcc
-CFLAGS = -pthread -O2
+CFLAGS = -pthread -O2 -I..
 LDFLAGS = -lrt
-COMMON = ./source/extra.h
+COMMON = ../extra.h
 
 # Handle cases where we're also profiling with the MMDC on the i.MX6Q
-ifneq ($(shell grep "define MMDC 1" source/extra.h),)
+ifneq ($(shell grep "define MMDC 1" ../extra.h),)
         COMMON += /media/speedy/litmus/tools/mmdc/mmdc.c
 endif
 
 # Include all the LITMUS^RT headers if we're using it
-ifneq ($(shell grep "define LITMUS 1" source/extra.h),)
+ifneq ($(shell grep "define LITMUS 1" ../extra.h),)
         CFLAGS += -I${LIBLITMUS}/include -I${LIBLITMUS}/arch/arm/include
         LDFLAGS += -L${LIBLITMUS} -llitmus
 endif
 
 all: bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec
 
+debug: all
+debug: CFLAGS += -ggdb3
+
 .PHONY: clean
 clean:
-        rm bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec
+        rm -f bin/cjpeg_wrbmp bin/huff_enc bin/gsm_enc bin/dijkstra bin/h264_dec bin/susan bin/adpcm_enc bin/rijndael_dec bin/huff_dec bin/rijndael_enc bin/gsm_dec bin/anagram bin/epic bin/ammunition bin/g723_enc bin/ndes bin/petrinet bin/statemate bin/cjpeg_transupp bin/mpeg2 bin/fmref bin/audiobeam bin/adpcm_dec
 
 bin/cjpeg_wrbmp: ${COMMON} ./source/cjpeg_wrbmp/cjpeg_wrbmp.c ./source/cjpeg_wrbmp/input.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
diff --git a/baseline/source/adpcm_dec/adpcm_dec.c b/baseline/source/adpcm_dec/adpcm_dec.c
index 04a5746..368e98d 100644
--- a/baseline/source/adpcm_dec/adpcm_dec.c
+++ b/baseline/source/adpcm_dec/adpcm_dec.c
@@ -35,7 +35,7 @@
   Forward declaration of functions
 */
 
-#include "../extra.h"
+#include "extra.h"
 
 void adpcm_dec_decode( int );
 int adpcm_dec_filtez( int *bpl, int *dlt );
diff --git a/baseline/source/adpcm_enc/adpcm_enc.c b/baseline/source/adpcm_enc/adpcm_enc.c
index 464768f..777aaf5 100644
--- a/baseline/source/adpcm_enc/adpcm_enc.c
+++ b/baseline/source/adpcm_enc/adpcm_enc.c
@@ -29,7 +29,7 @@
 
 /* common sampling rate for sound cards on IBM/PC */
 
-#include "../extra.h"
+#include "extra.h"
 #define SAMPLE_RATE 11025
 
 #define PI 3141
diff --git a/baseline/source/ammunition/ammunition.c b/baseline/source/ammunition/ammunition.c
index 269f4c0..052520e 100644
--- a/baseline/source/ammunition/ammunition.c
+++ b/baseline/source/ammunition/ammunition.c
@@ -24,7 +24,7 @@
 
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "bits.h"
 #include "arithm.h"
 #include "ammunition_stdlib.h"
diff --git a/baseline/source/anagram/anagram.c b/baseline/source/anagram/anagram.c
index b458fd2..5c1f29a 100644
--- a/baseline/source/anagram/anagram.c
+++ b/baseline/source/anagram/anagram.c
@@ -157,7 +157,7 @@
    steps to FindAnagram.
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "anagram_ctype.h"
 #include "anagram_stdlib.h"
 #include "anagram_strings.h"
diff --git a/baseline/source/audiobeam/audiobeam.c b/baseline/source/audiobeam/audiobeam.c
index 208de80..50ebfff 100644
--- a/baseline/source/audiobeam/audiobeam.c
+++ b/baseline/source/audiobeam/audiobeam.c
@@ -23,7 +23,7 @@
   Include section
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "audiobeamlibm.h"
 #include "audiobeamlibmalloc.h"
 #include "audiobeam.h"
diff --git a/baseline/source/cjpeg_transupp/cjpeg_transupp.c b/baseline/source/cjpeg_transupp/cjpeg_transupp.c
index 3f48539..5ec7e5e 100644
--- a/baseline/source/cjpeg_transupp/cjpeg_transupp.c
+++ b/baseline/source/cjpeg_transupp/cjpeg_transupp.c
@@ -29,7 +29,7 @@
   Include section
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "jpeglib.h"
 
 
diff --git a/baseline/source/cjpeg_wrbmp/cjpeg_wrbmp.c b/baseline/source/cjpeg_wrbmp/cjpeg_wrbmp.c
index 278725c..3c8d7ec 100644
--- a/baseline/source/cjpeg_wrbmp/cjpeg_wrbmp.c
+++ b/baseline/source/cjpeg_wrbmp/cjpeg_wrbmp.c
@@ -24,7 +24,7 @@
 
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "cdjpeg.h"
 
 #ifdef CJPEG_WRBMP_BMP_SUPPORTED
diff --git a/baseline/source/dijkstra/dijkstra.c b/baseline/source/dijkstra/dijkstra.c
index 1b6a52f..333fd43 100644
--- a/baseline/source/dijkstra/dijkstra.c
+++ b/baseline/source/dijkstra/dijkstra.c
@@ -17,7 +17,7 @@
 
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "input.h"
 
 /*
diff --git a/baseline/source/epic/epic.c b/baseline/source/epic/epic.c
index e258a4a..a1e344c 100644
--- a/baseline/source/epic/epic.c
+++ b/baseline/source/epic/epic.c
@@ -35,7 +35,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 #include "epic.h"
 
 #define X_SIZE  64
diff --git a/baseline/source/fmref/fmref.c b/baseline/source/fmref/fmref.c
index bcc7ce6..e4c9aaa 100644
--- a/baseline/source/fmref/fmref.c
+++ b/baseline/source/fmref/fmref.c
@@ -5,7 +5,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 #include "wcclibm.h"
 #ifndef M_PI
 #define M_PI        3.1415926535897932384626433832795
diff --git a/baseline/source/g723_enc/g723_enc.c b/baseline/source/g723_enc/g723_enc.c
index 331c821..209e3ce 100644
--- a/baseline/source/g723_enc/g723_enc.c
+++ b/baseline/source/g723_enc/g723_enc.c
@@ -31,7 +31,7 @@
    included in this Recommendation.
 */
 
-#include "../extra.h"
+#include "extra.h"
 struct g723_enc_state_t {
   long yl;  /* Locked or steady state step size multiplier. */
   short yu; /* Unlocked or non-steady state step size multiplier. */
diff --git a/baseline/source/gsm_dec/gsm_dec.c b/baseline/source/gsm_dec/gsm_dec.c
index 06d24d5..ba9a9e9 100644
--- a/baseline/source/gsm_dec/gsm_dec.c
+++ b/baseline/source/gsm_dec/gsm_dec.c
@@ -21,7 +21,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 #include "gsm.h"
 #include "add.h"
 #include "data.h"
diff --git a/baseline/source/gsm_enc/gsm_enc.c b/baseline/source/gsm_enc/gsm_enc.c
index 3f051ec..e8154bb 100644
--- a/baseline/source/gsm_enc/gsm_enc.c
+++ b/baseline/source/gsm_enc/gsm_enc.c
@@ -5,7 +5,7 @@
  * details.  THERE IS ABSOLUTELY NO WARRANTY FOR THIS SOFTWARE.
  */
 
-#include "../extra.h"
+#include "extra.h"
 #include "private.h"
 
 /*
diff --git a/baseline/source/h264_dec/h264_dec.c b/baseline/source/h264_dec/h264_dec.c
index 1fed68b..e74bec9 100644
--- a/baseline/source/h264_dec/h264_dec.c
+++ b/baseline/source/h264_dec/h264_dec.c
@@ -25,7 +25,7 @@
   Include section
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "h264_dec.h"
 
 
diff --git a/baseline/source/huff_dec/huff_dec.c b/baseline/source/huff_dec/huff_dec.c
index b83b99d..58b40d3 100644
--- a/baseline/source/huff_dec/huff_dec.c
+++ b/baseline/source/huff_dec/huff_dec.c
@@ -49,7 +49,7 @@ new schemes (this item is not a must)
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 typedef struct s_tree {
   unsigned int byte; /* A byte has to be coded as an unsigned integer to
                         allow a node to have a value over 255 */
diff --git a/baseline/source/huff_enc/huff_enc.c b/baseline/source/huff_enc/huff_enc.c
index 133fe95..0d74916 100644
--- a/baseline/source/huff_enc/huff_enc.c
+++ b/baseline/source/huff_enc/huff_enc.c
@@ -50,7 +50,7 @@ new schemes (this item is not a must)
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 typedef struct huff_enc_s_tree {
   unsigned int byte; /* A byte has to be coded as an unsigned integer to
                         allow a node to have a value over 255 */
diff --git a/baseline/source/litmusStuff.h b/baseline/source/litmusStuff.h
deleted file mode 100644
index dca2360..0000000
--- a/baseline/source/litmusStuff.h
+++ /dev/null
@@ -1,80 +0,0 @@
-#include <time.h>
-#include <sys/mman.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <signal.h>
-#include <limits.h>
-#include <../litmus.h>
-
-#define L3_CACHE_SIZE (11264*1024)
-
-#define SAVE_RESULTS if(jobsComplete>-1) progTime[jobsComplete]=(end.tv_nsec-start.tv_nsec)+(1000000000*(end.tv_sec-start.tv_sec));
-
-
-#define SET_UP char *thisProgram=argv[1];\
-        int maxJobs=atoi(argv[2]);\
-        char *thisCore=argv[3];\
-        char *otherCore=argv[4];\
-        char *otherProgram=argv[5];\
-        char *runID=argv[6];\
-        int output=atoi(argv[7]);\
-        struct timespec start, end;\
-        int jobsComplete;\
-        long progTime[maxJobs*output];\
-        char fileName[50];\
-        char *bigArray;\
-        strcpy(fileName, runID);\
-        strcat(fileName, ".txt");\
-        mlockall(MCL_CURRENT || MCL_FUTURE);
-
-#define WRITE_TO_FILE if (output){\
-        munlockall();\
-        FILE *fp=fopen(fileName, "a");\
-        for(jobsComplete=0; jobsComplete<maxJobs; jobsComplete++){\
-                fprintf(fp, "%s %s %s %s %d %ld %s %d \n",\
-                thisProgram, otherProgram, thisCore, otherCore, maxJobs,\
-                progTime[jobsComplete], runID, jobsComplete);\
-        }\
-        fclose(fp);\
-        }
-
-#define KILL_CACHE bigArray=(char *)malloc(L3_CACHE_SIZE);\
-if (bigArray==NULL) perror("Malloc failed.\n");\
-memset(bigArray, 1, L3_CACHE_SIZE);\
-munlock(bigArray, L3_CACHE_SIZE);\
-free(bigArray);\
-bigArray=NULL;
-
-
-
-//invoke start timer twice, stop timer to make sure timer and vars are in cache
-#define START_TIMER clock_gettime(CLOCK_MONOTONIC, &start);\
-        clock_gettime(CLOCK_MONOTONIC, &end);\
-        clock_gettime(CLOCK_MONOTONIC, &start);\
-                
-#define STOP_TIMER clock_gettime(CLOCK_MONOTONIC, &end);
-
-
-#define START_LOOP START_TIMER
-
-#define STOP_LOOP if (output) {STOP_TIMER SAVE_RESULTS} sleep_next_period();
-
-
-/*
-Intended structure
-
-main
-SET_UP
-notice that STOP LOOP negates the ++ if outout=0
-for (jobsComplete=-1; jobsComplete<maxJobs; jobsComplete++){
-                KILL_CACHE
-                START_TIMER
-                tacleInit();
-                tacleMain();
-                STOP_TIMER
-                SAVE_RESULTS
-}
-WRITE_TO_FILE
-tacleReturn     
-*/
diff --git a/baseline/source/mpeg2/mpeg2.c b/baseline/source/mpeg2/mpeg2.c
index a810744..0567225 100644
--- a/baseline/source/mpeg2/mpeg2.c
+++ b/baseline/source/mpeg2/mpeg2.c
@@ -48,7 +48,7 @@
 /*
   Forward declaration of data types
 */
-#include "../extra.h"
+#include "extra.h"
 struct mbinfo;
 
 
diff --git a/baseline/source/ndes/ndes.c b/baseline/source/ndes/ndes.c
index b0f494c..f169c62 100644
--- a/baseline/source/ndes/ndes.c
+++ b/baseline/source/ndes/ndes.c
@@ -17,7 +17,7 @@
 */
 
 /* A read from this address will result in an known value of 1 */
-#include "../extra.h"
+#include "extra.h"
 #define KNOWN_VALUE 1
 #define NDES_WORSTCASE 1
 /*
diff --git a/baseline/source/petrinet/petrinet.c b/baseline/source/petrinet/petrinet.c
index cabb40d..6ef3392 100644
--- a/baseline/source/petrinet/petrinet.c
+++ b/baseline/source/petrinet/petrinet.c
@@ -25,7 +25,7 @@
 */
 
 
-#include "../extra.h"
+#include "extra.h"
 
 #ifdef PROFILING
 #include <stdio.h>
diff --git a/baseline/source/rijndael_dec/rijndael_dec.c b/baseline/source/rijndael_dec/rijndael_dec.c
index 93bb424..61db767 100644
--- a/baseline/source/rijndael_dec/rijndael_dec.c
+++ b/baseline/source/rijndael_dec/rijndael_dec.c
@@ -38,7 +38,7 @@
   -----------------------------------------------------------------------
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "aes.h"
 #include "rijndael_dec_libc.h"
 
diff --git a/baseline/source/rijndael_enc/rijndael_enc.c b/baseline/source/rijndael_enc/rijndael_enc.c
index 0c10353..6c85eee 100644
--- a/baseline/source/rijndael_enc/rijndael_enc.c
+++ b/baseline/source/rijndael_enc/rijndael_enc.c
@@ -38,7 +38,7 @@
   -----------------------------------------------------------------------
 */
 
-#include "../extra.h"
+#include "extra.h"
 #include "aes.h"
 #include "rijndael_enc_libc.h"
 
diff --git a/baseline/source/statemate/statemate.c b/baseline/source/statemate/statemate.c
index 00491e5..888d0fa 100644
--- a/baseline/source/statemate/statemate.c
+++ b/baseline/source/statemate/statemate.c
@@ -27,7 +27,7 @@
   Macro definitions
 */
 
-#include "../extra.h"
+#include "extra.h"
 
 #define SYS_bit_get(a,b) (a)[(b)]
 #define SYS_bit_clr(a,b) (a)[(b)] = 0
diff --git a/baseline/source/susan/susan.c b/baseline/source/susan/susan.c
index 4bc4cb8..dc27fb1 100644
--- a/baseline/source/susan/susan.c
+++ b/baseline/source/susan/susan.c
@@ -269,7 +269,7 @@
 
 \**********************************************************************/
 
-#include "../extra.h"
+#include "extra.h"
 #include "wcclibm.h"
 #include "wccfile.h"
 #include "wccmalloc.h"
diff --git a/dis/Makefile b/dis/Makefile
index 2c1ff89..df93116 100755
--- a/dis/Makefile
+++ b/dis/Makefile
@@ -4,34 +4,33 @@
 # that a good Makefile should follow. Sorry - at least I have one.)
 LIBLITMUS ?= /media/speedy/litmus/liblitmus
 CC ?= gcc
-override CFLAGS += -pthread -O2
+CFLAGS += -pthread -O2 -I..
 LDFLAGS = -lrt -lm
-
-# We need a different include path depending on the target, so use a
-# target-specific variable
-field matrix neighborhood pointer transitive update random_walk: override CFLAGS += -I../baseline/source
-field matrix neighborhood pointer transitive update random_walk: COMMON = ../baseline/source/extra.h
-field_all matrix_all neighborhood_all pointer_all transitive_all update_all: override CFLAGS += -I../all_pairs/source
-field_all matrix_all neighborhood_all pointer_all transitive_all update_all: COMMON = ../all_pairs/source/extra.h
+COMMON = ../extra.h
 
 # Handle cases where we're also profiling with the MMDC on the i.MX6Q
-ifneq ($(shell grep "define MMDC 1" ../baseline/source/extra.h),)
+ifneq ($(shell grep "define MMDC 1" ../extra.h),)
         COMMON += /media/speedy/litmus/tools/mmdc/mmdc.c
 endif
 
 # Include all the LITMUS^RT headers if we're using it
-ifneq ($(shell grep "define LITMUS 1" ../baseline/source/extra.h),)
+ifneq ($(shell grep "define LITMUS 1" ../extra.h),)
         CFLAGS += -I${LIBLITMUS}/include -I${LIBLITMUS}/arch/arm/include
         LDFLAGS += -L${LIBLITMUS} -llitmus
 endif
 
 all: baseline
+debug: baseline
+debug_pairs: pairs
+debug debug_pairs: CFLAGS += -ggdb3
 baseline: field matrix neighborhood pointer transitive update random_walk
 pairs: field_all matrix_all neighborhood_all pointer_all transitive_all update_all
 
 .PHONY: clean
 clean:
-        rm field_all matrix_all neighborhood_all pointer_all transitive_all update_all field matrix neighborhood pointer transitive update random_walk
+        rm -f field_all matrix_all neighborhood_all pointer_all \
+      transitive_all update_all field matrix neighborhood \
+      pointer transitive update random_walk
 
 field: ${COMMON} ./Field/field.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
@@ -49,14 +48,14 @@ random_walk: ${COMMON} random_walk.c
         $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
 
 field_all: ${COMMON} ./Field/field.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+        $(CC) $(CFLAGS) -DPAIRED $^ -o $@ $(LDFLAGS)
 matrix_all: ${COMMON} ./Matrix/ver2/matrix.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+        $(CC) $(CFLAGS) -DPAIRED $^ -o $@ $(LDFLAGS)
 neighborhood_all: ${COMMON} ./Neighborhood/neighborhood.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+        $(CC) $(CFLAGS) -DPAIRED $^ -o $@ $(LDFLAGS)
 pointer_all: ${COMMON} ./Pointer/pointer.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+        $(CC) $(CFLAGS) -DPAIRED $^ -o $@ $(LDFLAGS)
 transitive_all: ${COMMON} ./Transitive/transitive.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+        $(CC) $(CFLAGS) -DPAIRED $^ -o $@ $(LDFLAGS)
 update_all: ${COMMON} ./Update/update.c
-        $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+        $(CC) $(CFLAGS) -DPAIRED $^ -o $@ $(LDFLAGS)
diff --git a/baseline/source/extra.h b/extra.h
index e8f3d18..3215951 100644
--- a/baseline/source/extra.h
+++ b/extra.h
@@ -2,18 +2,18 @@
  * Copyright 2019 Sims Hill Osborne and Joshua Bakita
  *
  * This header provides facilities by which to separably run and time TACLeBench
+ * To use this for paired task timing, define PAIRED (pass CFLAGS=-DPAIRED to make)
  **/
 #define _GNU_SOURCE
-#include <time.h>
-#include <sys/mman.h>
-#include <stdlib.h>
+#include <fcntl.h>     // For O_CREAT and O_RDWR
+#include <sched.h>     // For sched_yield()
+#include <semaphore.h> // For sem_{open, post, wait}()
 #include <stdio.h>
-#include <string.h>
-#include <signal.h>
-#include <limits.h>
-#include <fcntl.h>
-#include <stdint.h>
-#include <sched.h>
+#include <stdlib.h>    // For exit()
+#include <string.h>    // For strlen()
+#include <sys/mman.h>  // For mlockall()
+#include <unistd.h>    // For ftruncate()
+#include <time.h>
 
 // This is only visible if _GNU_SOURCE is defined, and that define does not
 // come along to places where this file is included. Address this by manually
@@ -55,7 +55,13 @@ extern int sched_getcpu();
 
 // Store state globally so that the job can be outside main()
 // Arrays use float as a comprimise between overflow and size
+// Paired arrays use long longs as precision is more important for those times
+#ifdef PAIRED
+long long *_rt_start_time;
+long long *_rt_end_time;
+#else
 float *_rt_exec_time;
+#endif
 #if MMDC_PERF
 float *_rt_mmdc_read;
 float *_rt_mmdc_write;
@@ -68,36 +74,103 @@ struct timespec _rt_start, _rt_end;
 
 char *_rt_run_id;
 char *_rt_our_prog_name;
+char *_rt_other_prog_name;
+char *_rt_other_core;
 #define _RT_FILENAME_LEN 64
+#define _BILLION (1000*1000*1000)
+#ifdef PAIRED
+char *_rt_barrier;
+sem_t *_rt_first_sem, *_rt_second_sem;
+int _rt_lock_id;
+#endif
 
-#define LOAD_PARAMS_ITRL \
-    if (argc != 6) { \
-        fprintf(stderr, "Usage: %s <name> <loops> <my core> <runID> <save results?>\n", argv[0]);\
-        fprintf(stderr, " <loops> integer number of iterations. -1 for infitite.\n");\
-        fprintf(stderr, " <save results?> 1 to save results, 0 to discard.\n");\
-        fprintf(stderr, " <my core> UNUSED. Core is now auto-detected.\n");\
-        exit(1);\
-    }\
-    _rt_our_prog_name = argv[1];\
-    _rt_max_jobs = atol(argv[2]);\
-    _rt_core = sched_getcpu();\
-    _rt_run_id = argv[4];\
-    _rt_will_output = atoi(argv[5]);\
-    if (_rt_max_jobs < 0 && _rt_will_output != 0) {\
-        fprintf(stderr, "Infinite loops only supported when _rt_will_output is disabled!\n");\
-        exit(1);\
-    }\
-    if (strlen(_rt_run_id) + 5 > _RT_FILENAME_LEN) {\
-        fprintf(stderr, "Run ID is too large! Keep it to less than %d characters.\n", _RT_FILENAME_LEN);\
-        exit(1);\
-    }\
-    _rt_exec_time = calloc(_rt_max_jobs * _rt_will_output, sizeof(float));\
-    if (!_rt_exec_time) {\
-        perror("Unable to allocate buffer for execution times");\
-        exit(1);\
-    }\
-    _rt_jobs_complete = 0;\
+static void _rt_load_params_itrl(int argc, char **argv) {
+#ifdef PAIRED
+    if (argc != 8) {
+        fprintf(stderr, "Usage: %s <name> <loops> <my core> <other core> <other name> <runID> <lockID>", argv[0]);
+        fprintf(stderr, " <name> string for logging. Name of this task.\n");
+        fprintf(stderr, " <loops> integer number of iterations. -1 for infinite.\n");
+        fprintf(stderr, " <my core> UNUSED. Core is now auto-detected.\n");
+        fprintf(stderr, " <other core> integer for logging. Core of paired task.\n");
+        fprintf(stderr, " <other name> string for logging. Name of paired task.\n");
+        fprintf(stderr, " <runID> string to append with .txt to yield output file name.\n");
+        fprintf(stderr, " <lockID> 1 to indicate this is pair member 1, otherwise pair member 2.\n");
+        exit(1);
+    }
+#else
+    if (argc != 6) {
+        fprintf(stderr, "Usage: %s <name> <loops> <my core> <runID> <save results?>\n", argv[0]);
+        fprintf(stderr, " <name> string for logging. Name of this task.\n");
+        fprintf(stderr, " <loops> integer number of iterations. -1 for infinite.\n");
+        fprintf(stderr, " <my core> UNUSED. Core is now auto-detected.\n");
+        fprintf(stderr, " <runID> string to append with .txt to yield output file name.\n");
+        fprintf(stderr, " <save results?> 1 to save results, 0 to discard.\n");
+        exit(1);
+    }
+#endif
+    _rt_our_prog_name = argv[1];
+    _rt_max_jobs = atol(argv[2]);
+    _rt_core = sched_getcpu();
+#ifdef PAIRED
+    _rt_other_core = argv[4];
+    _rt_other_prog_name = argv[5];
+    _rt_run_id = argv[6];
+    _rt_lock_id = atoi(argv[7]);
+    // The paired version doesn't support disabling output (legacy compatibility)
+    _rt_will_output = 1;
+#else
+    _rt_other_core = "none";
+    _rt_other_prog_name = "none";
+    _rt_run_id = argv[4];
+    _rt_will_output = atoi(argv[5]);
+#endif /* PAIRED */
+    if (_rt_max_jobs < 0 && _rt_will_output != 0) {
+        fprintf(stderr, "Infinite loops only supported when _rt_will_output is disabled!\n");
+        exit(1);
+    }
+    if (strlen(_rt_run_id) + 5 > _RT_FILENAME_LEN) {
+        fprintf(stderr, "Run ID is too large! Keep it to less than %d characters.\n", _RT_FILENAME_LEN);
+        exit(1);
+    }
+#ifdef PAIRED
+    _rt_start_time = calloc(_rt_max_jobs * _rt_will_output, sizeof(long long));
+    _rt_end_time = calloc(_rt_max_jobs * _rt_will_output, sizeof(long long));
+    if (!_rt_end_time || !_rt_start_time) {
+        perror("Unable to allocate buffers for execution times");
+        exit(1);
+    }
+    _rt_first_sem = sem_open("/_libextra_first_sem", O_CREAT, 644, 0);
+    _rt_second_sem = sem_open("/_libextra_second_sem", O_CREAT, 644, 0);
+    if (_rt_first_sem == SEM_FAILED || _rt_second_sem == SEM_FAILED) {
+        perror("Error while creating semaphores");
+        exit(1);
+    }
+    int barrier_file = shm_open("/_libextra_barrier", O_CREAT | O_RDWR, 644);
+    if (barrier_file == -1) {
+        perror("Error while creating shared memory for barrier synchronization");
+        exit(1);
+    }
+    if (ftruncate(barrier_file, 1) == -1) {
+        perror("Error while setting size of shared memory for barrier synchronization");
+        exit(1);
+    }
+    _rt_barrier = mmap(NULL, 1, PROT_WRITE, MAP_SHARED, barrier_file, 0);
+    if (_rt_barrier == MAP_FAILED) {
+        perror("Error while mapping shared memory for barrier synchronization");
+        exit(1);
+    }
+    *_rt_barrier = 0;
+#else
+    _rt_exec_time = calloc(_rt_max_jobs * _rt_will_output, sizeof(float));
+    if (!_rt_exec_time) {
+        perror("Unable to allocate buffer for execution times");
+        exit(1);
+    }
+#endif /* PAIRED */
+    _rt_jobs_complete = 0;
     mlockall(MCL_CURRENT || MCL_FUTURE);
+}
+#define LOAD_PARAMS_ITRL _rt_load_params_itrl(argc, argv);
 
 #define SETUP_MMDC \
     _rt_mmdc_read = calloc(_rt_max_jobs * _rt_will_output, sizeof(float));\
@@ -213,6 +286,44 @@ char *_rt_our_prog_name;
     fclose(fp);
 #endif
 
+// This semaphore-based synchronization is from Sims
+#define FIRST_UNLOCK \
+    if (_rt_lock_id == 1) {\
+        if (sem_post(_rt_second_sem) != 0) {\
+            perror("Unable to unlock second semaphore");\
+            exit(1);\
+        }\
+    } \
+    else {\
+        if (sem_post(_rt_first_sem) != 0) {\
+            perror("Unable to unlock first semaphore");\
+            exit(1);\
+        }\
+    } \
+
+#define FIRST_LOCK \
+    if (_rt_lock_id == 1) {\
+        if (sem_wait(_rt_first_sem) != 0) {\
+            perror("Unable to wait on first semaphore");\
+            exit(1);\
+        }\
+    }\
+    else {\
+        if (sem_wait(_rt_second_sem) != 0) {\
+            perror("Unable to wait on second semaphore");\
+            exit(1);\
+        }\
+    }
+
+// This ensures a very low difference between pair member start times
+#define BARRIER_SYNC \
+    if (__sync_bool_compare_and_swap(_rt_barrier, 0, 1)) {\
+        while (!__sync_bool_compare_and_swap(_rt_barrier, 0, 0)) {};\
+    }\
+    else {\
+        __sync_bool_compare_and_swap(_rt_barrier, 1, 0);\
+    }
+
 // Buffer timing result from a single job
 static void _rt_save_job_result() {
     if (_rt_jobs_complete >= _rt_max_jobs) {
@@ -220,9 +331,18 @@ static void _rt_save_job_result() {
         exit(1);
     }
     if (_rt_jobs_complete > -1 && _rt_will_output) {
+#ifdef PAIRED
+        _rt_start_time[_rt_jobs_complete] = _rt_start.tv_sec;
+        _rt_start_time[_rt_jobs_complete] *= _BILLION;
+        _rt_start_time[_rt_jobs_complete] += _rt_start.tv_nsec;
+        _rt_end_time[_rt_jobs_complete] = _rt_end.tv_sec;
+        _rt_end_time[_rt_jobs_complete] *= _BILLION;
+        _rt_end_time[_rt_jobs_complete] += _rt_end.tv_nsec;
+#else
         _rt_exec_time[_rt_jobs_complete] = _rt_end.tv_sec - _rt_start.tv_sec;
-        _rt_exec_time[_rt_jobs_complete] *= 1e9;
+        _rt_exec_time[_rt_jobs_complete] *= _BILLION;
         _rt_exec_time[_rt_jobs_complete] += _rt_end.tv_nsec - _rt_start.tv_nsec;
+#endif /* PAIRED */
 #if MMDC_PROF
         _rt_mmdc_read[_rt_jobs_complete] = mmdc_res.read_bytes;
         _rt_mmdc_write[_rt_jobs_complete] = mmdc_res.write_bytes;
@@ -234,31 +354,54 @@ static void _rt_save_job_result() {
 static void _rt_write_to_file() {
     char fileName[_RT_FILENAME_LEN];
     FILE *fp;
-    if (!_rt_will_output)
-        return;
     munlockall();
+    if (!_rt_will_output)
+        goto out;
     strcpy(fileName, _rt_run_id);
     strcat(fileName, ".txt");
     fp = fopen(fileName, "a");
     if (fp == NULL) {
-        perror("Unable to open _rt_will_output file");
+        perror("Unable to open output file");
         exit(1);
     }
-    // Same format as the paired results with "none" for unused fields
+    // Baseline output uses a similar format with "none" for unused fields
     for (int i = 0; i < _rt_jobs_complete; i++){
-        fprintf(fp, "%s none %u none %ld %.f %s %d %.f %.f \n",
-        _rt_our_prog_name, _rt_core, _rt_max_jobs,
-        _rt_exec_time[i], _rt_run_id, i,
+        fprintf(fp, "%s %s %u %s %ld", _rt_our_prog_name, _rt_other_prog_name,
+            _rt_core, _rt_other_core, _rt_max_jobs);
+#ifdef PAIRED
+        // For unclear legacy reasons, paired tasks emit sec and ns separately
+        fprintf(fp, " %lld %lld %lld %lld",
+            _rt_start_time[i] / _BILLION, _rt_start_time[i] % _BILLION,
+            _rt_end_time[i] / _BILLION, _rt_end_time[i] % _BILLION);
+#else
+        fprintf(fp, " %.f", _rt_exec_time[i]);
+#endif /* PAIRED */
+        fprintf(fp, " %s %d %.f %.f\n",  _rt_run_id, i,
 #if MMDC_PROF
-        _rt_mmdc_read[i], _rt_mmdc_write[i]);
+            _rt_mmdc_read[i], _rt_mmdc_write[i]);
 #else
-        0.0, 0.0);
-#endif
+            0.0, 0.0);
+#endif /* MMDC_PROF */
     }
     fclose(fp);
+out:
 #if LITMUS
     CLEANUP_LITMUS
-#endif
+#endif /* LITMUS */
+#ifdef PAIRED
+    munmap(_rt_barrier, 1);
+    shm_unlink("/_libextra_barrier");
+    sem_unlink("/_libextra_first_sem");
+    sem_unlink("/_libextra_second_sem");
+    free(_rt_start_time);
+    free(_rt_end_time);
+#else
+    free(_rt_exec_time);
+#endif /* PAIRED */
+#if MMDC_PROF
+    free(_rt_mmdc_read);
+    free(_rt_mmdc_write);
+#endif /* MMDC_PROF */
 }
 
 // Start a job
@@ -269,12 +412,19 @@ static void _rt_start_loop() {
     }
 #else
     sched_yield();
-#endif
+#endif /* LITMUS */
+#ifdef PAIRED
+    FIRST_UNLOCK
+    FIRST_LOCK
+#endif /* PAIRED */
     FLUSH_CACHES
+#ifdef PAIRED
+    BARRIER_SYNC
+#endif /* PAIRED */
 #if MMDC_PROF
     /* This disables profiling, resets the counters, clears the overflow bit, and enables profiling */
     start_mmdc_profiling(mmdc);
-#endif
+#endif /* MMDC_PROF */
     clock_gettime(CLOCK_MONOTONIC, &_rt_start);
 }
 
@@ -285,7 +435,7 @@ static void _rt_stop_loop() {
     /* This freezes the profiling and makes results available */
     pause_mmdc_profiling(mmdc);
     get_mmdc_profiling_results(mmdc, &mmdc_res);
-#endif
+#endif /* MMDC_PROF */
     _rt_save_job_result();
     _rt_jobs_complete++;
 }