Merged

9 files changed, 1515 insertions, 99 deletions

diff --git a/Makefile b/Makefile
index b135529..c2a1105 100644
--- a/Makefile
+++ b/Makefile
@@ -73,8 +73,9 @@ AR  := ${CROSS_COMPILE}${AR}
 
 all     = lib ${rt-apps}
 rt-apps = cycles base_task rt_launch rtspin release_ts measure_syscall \
-          base_mt_task uncache runtests resctrl mc2spin test1 mttest \
-          mc2thrash mc2shm mc2bench
+          base_mt_task uncache runtests resctrl mc2spin test1 \
+          mc2thrash mc2shm mc2thrash1 mc2thrash2 mc2thrash3 mc2bench\
+          mtdag
 
 .PHONY: all lib clean dump-config TAGS tags cscope help doc
 
@@ -253,15 +254,24 @@ lib-memthrash = -lrt
 obj-test1 = test1.o common.o
 lib-test1 = -lrt -static
 
-obj-mttest = mttest.o
-ldf-mttest = -pthread
-
 obj-mc2thrash = mc2thrash.o common.o
 lib-mc2thrash = -lrt -static
 
+obj-mc2thrash1 = mc2thrash.o common.o
+lib-mc2thrash1 = -lrt -static
+
+obj-mc2thrash2 = mc2thrash.o common.o
+lib-mc2thrash2 = -lrt -static
+
+obj-mc2thrash3 = mc2thrash.o common.o
+lib-mc2thrash3 = -lrt -static
+
 obj-mc2shm = mc2shm.o common.o
 lib-mc2shm = -lrt -static
 
+obj-mtdag = mtdag.o common.o
+ldf-mtdag = -lrt -pthread -static
+
 obj-mc2bench = mc2bench.o common.o
 lib-mc2bench = -lrt -static
 # ##############################################################################
diff --git a/bin/mc2bench.c b/bin/mc2bench.c
new file mode 100644
index 0000000..dfb4f15
--- /dev/null
+++ b/bin/mc2bench.c
@@ -0,0 +1,503 @@
+#include <sys/time.h>
+#include <sys/mman.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <string.h>
+#include <assert.h>
+#include <limits.h>
+#include <fcntl.h>
+
+#include "litmus.h"
+#include "common.h"
+#include "color_shm.h"
+
+#define PAGE_SIZE (4096)
+//#define CACHELINE_SIZE 32
+//#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int))
+//#define CACHELINES_IN_1KB (1024 / sizeof(cacheline_t))
+#define INTS_IN_1KB     (1024 / sizeof(int))
+
+//typedef struct cacheline
+//{
+//        int line[INTS_IN_CACHELINE];
+//} __attribute__((aligned(CACHELINE_SIZE))) cacheline_t;
+
+static int loops = 100000;
+static cacheline_t* arena = NULL;
+static cacheline_t* local_buf = NULL;
+
+struct timeval t1,t2;
+
+lt_t exectime[100000];
+
+inline unsigned long get_cyclecount (void)
+{
+        unsigned long value;
+        // Read CCNT Register
+        asm volatile ("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value));
+        return value;
+}
+
+#define UNCACHE_DEV "/dev/litmus/uncache"
+
+static cacheline_t* alloc_shm(int cs, size_t size, int use_huge_pages, int use_uncache_pages)
+{
+        struct color_ioctl_cmd shm_info;
+        struct color_ioctl_offset shm_offset;
+        cacheline_t* arena = NULL;
+        
+
+        shm_info.color = 0x0000ffff; 
+        
+        if (cs == 1 || cs == 2)
+                shm_info.bank  = 0x00000020; // hi crit. bank
+        else if (cs == 3)
+                shm_info.bank  = 0x00000040; // levelC
+        
+        shm_offset.offset = 0;
+        shm_offset.lock = 1;
+
+        arena = color_mmap(size, shm_info, shm_offset);
+        if (arena == MAP_FAILED) {
+                printf("mmap failed.\n");
+                return NULL;
+        }
+
+        mlockall(MCL_CURRENT | MCL_FUTURE);
+        /* finish allocation */
+
+    assert(arena);
+
+    return arena;
+}
+
+static cacheline_t* alloc_local(size_t size, int use_huge_pages, int use_uncache_pages)
+{
+    int flags = MAP_PRIVATE | MAP_POPULATE;
+    cacheline_t* arena = NULL;
+    int fd;
+
+    if(use_huge_pages)
+        flags |= MAP_HUGETLB;
+
+        if(use_uncache_pages) {
+                        fd = open(UNCACHE_DEV, O_RDWR|O_SYNC);
+                        if (fd == -1)
+                                        bail_out("Failed to open uncache device. Are you running the LITMUS^RT kernel?");
+        }
+        else {
+                        fd = -1;
+                        flags |= MAP_ANONYMOUS;
+        }
+
+    arena = mmap(0, size, PROT_READ | PROT_WRITE, flags, fd, 0);
+        
+    if(use_uncache_pages)
+                close(fd);
+
+    assert(arena);
+
+    return arena;
+}
+
+static void dealloc_arena(cacheline_t* arena, size_t size)
+{
+                int ret = munmap((void*)arena, size);
+        if(ret != 0)
+                bail_out("munmap() error");
+}
+
+static int randrange(int min, int max)
+{
+        /* generate a random number on the range [min, max) w/o skew */
+        int limit = max - min;
+        int devisor = RAND_MAX/limit;
+        int retval;
+
+        do {
+                retval = rand() / devisor;
+        } while(retval == limit);
+        retval += min;
+
+        return retval;
+}
+
+static void init_arena(cacheline_t* arena, size_t size)
+{
+    int i;
+        size_t num_arena_elem = size / sizeof(cacheline_t);
+
+        /* Generate a cycle among the cache lines using Sattolo's algorithm.
+           Every int in the cache line points to the same cache line.
+           Note: Sequential walk doesn't care about these values. */
+        for (i = 0; i < num_arena_elem; i++) {
+                int j;
+                for(j = 0; j < INTS_IN_CACHELINE; ++j)
+                        arena[i].line[j] = i;
+        }
+        while(1 < i--) {
+                int j = randrange(0, i);
+                cacheline_t temp = arena[j];
+                arena[j] = arena[i];
+                arena[i] = temp;
+        }
+}
+
+/* Random walk around the arena in cacheline-sized chunks.
+   Cacheline-sized chucks ensures the same utilization of each
+   hit line as sequential read. (Otherwise, our utilization
+   would only be 1/INTS_IN_CACHELINE.) */
+static int random_walk(cacheline_t *mem, int wss, int write_cycle)
+{
+        /* a random cycle among the cache lines was set up by init_arena(). */
+        int sum, i, next, j;
+
+        int numlines = wss * CACHELINES_IN_1KB;
+
+        sum = 0;
+
+        /* contents of arena is structured s.t. offsets are all
+           w.r.t. to start of arena, so compute the initial offset */
+        next = mem - arena;
+
+        if (write_cycle == 0) {
+                for (i = 0; i < numlines; i++) {
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                /* every element in the cacheline has the same value */
+                                //next = arena[next].line[j];
+                                local_buf[next].line[j] = arena[next].line[j];
+                                //sum += next;
+                                //local_buf[next].line[j] = next;
+                        }
+                }
+        }
+        
+        else {
+                int w, which_line;
+                for (i = 0, w = 0; i < numlines; i++) {
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                which_line = next;
+                                next = local_buf[next].line[j];
+                                ((volatile cacheline_t*)arena)[which_line].line[j] = next;
+                                //sum += next;
+                        }
+                }
+        }
+        return sum;
+}
+
+static cacheline_t* random_start(int wss)
+{
+        return arena + randrange(0, ((wss * 1024)/sizeof(cacheline_t)));
+}
+
+static int sequential_walk(cacheline_t *mem, int wss, int write_cycle)
+{
+    int sum = 0, i, j;
+    //int* mem = (int*)_mem; /* treat as raw buffer of ints */
+    int num_ints = wss * CACHELINES_IN_1KB;
+
+    if (write_cycle > 0) {
+        for (i = 0; i < num_ints; i++) {
+            //if (i % write_cycle == (write_cycle - 1)) {
+                                //mem[i]++;
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                //sum += local_buf[i].line[j];
+                                mem[i].line[j] = local_buf[i].line[j];;
+                        }
+                        //}
+            //else {
+            //    sum += mem[i];
+                        //}
+        }
+    } else {
+        /* sequential access, pure read */
+        for (i = 0; i < num_ints; i++) {
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                //sum += mem[i];
+                                local_buf[i].line[j] = mem[i].line[j];
+                                //sum += mem[i];
+                        }
+                }
+    }
+    return sum;
+}
+
+static cacheline_t* sequential_start(int wss)
+{
+        return arena;
+/*    static int pos = 0;
+
+    int num_cachelines = wss * CACHELINES_IN_1KB;
+        size_t num_arena_elem = wss * 1024 / sizeof(cacheline_t);
+    cacheline_t *mem;
+*/      
+        
+    /* Don't allow re-use between allocations.
+     * At most half of the arena may be used
+     * at any one time.
+     */
+/*    if (num_cachelines * 2 > num_arena_elem)
+        bail_out("static memory arena too small");
+
+    if (pos + num_cachelines > num_arena_elem) {
+        mem = arena;
+        pos = num_cachelines;
+    } else {
+        mem = arena + pos;
+        pos += num_cachelines;
+    }
+
+    return mem;*/
+}
+
+static volatile int dont_optimize_me = 0;
+
+static void usage(char *error) {
+        fprintf(stderr, "Error: %s\n", error);
+        fprintf(stderr,
+                "Usage:\n"
+                "       rt_spin [COMMON-OPTS] WCET PERIOD DURATION\n"
+                "       rt_spin [COMMON-OPTS] -f FILE [-o COLUMN] WCET PERIOD\n"
+                "       rt_spin -l\n"
+                "\n"
+                "COMMON-OPTS = [-w] [-s SCALE]\n"
+                "              [-p PARTITION/CLUSTER [-z CLUSTER SIZE]] [-m CRITICALITY LEVEL]\n"
+                "              [-k WSS] [-l LOOPS] [-b BUDGET]\n"
+                "\n"
+                "WCET and PERIOD are milliseconds, DURATION is seconds.\n");
+        exit(EXIT_FAILURE);
+}
+
+static int loop_once(int cs, int wss, unsigned int iter)
+{
+        //cacheline_t *mem;
+        //int temp;
+        
+        //mem = random_start(wss);
+        //temp = random_walk(mem, wss, 1);
+    
+        //mem = sequential_start(wss);
+        //temp = sequential_walk(arena, wss, 1);
+        run_bench(cs, wss, arena, local_buf, &exectime[iter]);
+        dont_optimize_me = (int)exectime[iter];
+        
+        return dont_optimize_me;
+}
+
+static int job(int cs, int wss, double exec_time, double program_end)
+{
+        if (wctime() > program_end)
+                return 0;
+        else {
+                register unsigned long t;
+                register unsigned int iter = 0;
+                //t = get_cyclecount();
+                //gettimeofday(&t1, NULL);
+                while(iter < loops) {
+                        loop_once(cs, wss, iter++);
+                }
+                //gettimeofday(&t2, NULL);
+                //printf("%ld cycles\n", get_cyclecount() - t);
+                //printf("%ld\n", ((t2.tv_sec * 1000000 + t2.tv_usec) - (t1.tv_sec * 1000000 + t1.tv_usec)));
+                sleep_next_period(); 
+        return 1;
+        }
+}
+
+#define OPTSTR "p:wl:m:i:b:k:c:"
+int main(int argc, char** argv)
+{
+        int ret, i;
+        lt_t wcet, period, budget;
+        double wcet_ms, period_ms, budget_ms;
+        unsigned int priority = LITMUS_NO_PRIORITY;
+        int migrate = 0;
+        int cluster = 0;
+        int opt;
+        int wait = 0;
+        double duration = 0, start = 0;
+        struct rt_task param;
+        struct mc2_task mc2_param;
+        struct reservation_config config;
+        int res_type = PERIODIC_POLLING;
+        size_t arena_sz;
+        int wss, cs;
+
+        /* default for reservation */
+        config.id = 0;
+        config.priority = LITMUS_NO_PRIORITY; /* use EDF by default */
+        config.cpu = -1;
+        
+        mc2_param.crit = CRIT_LEVEL_C;
+        
+        budget_ms = 1000;
+        
+        while ((opt = getopt(argc, argv, OPTSTR)) != -1) {
+                switch (opt) {
+                case 'w':
+                        wait = 1;
+                        break;
+                case 'p':
+                        cluster = atoi(optarg);
+                        migrate = 1;
+                        config.cpu = cluster;
+                        break;
+                case 'l':
+                        loops = atoi(optarg);
+                        break;
+                case 'k':
+                        wss = atoi(optarg);
+                        break;
+                case 'm':
+                        mc2_param.crit = atoi(optarg);
+                        if ((mc2_param.crit >= CRIT_LEVEL_A) && (mc2_param.crit <= CRIT_LEVEL_C)) {
+                                res_type = PERIODIC_POLLING;
+                        }
+                        else
+                                usage("Invalid criticality level.");
+                        break;
+                case 'b':
+                        budget_ms = atof(optarg);
+                        break;
+                case 'i':
+                        config.priority = atoi(optarg);
+                        break;
+                case 'c':
+                        cs = atoi(optarg);
+                        break;
+                case ':':
+                        usage("Argument missing.");
+                        break;
+                case '?':
+                default:
+                        usage("Bad argument.");
+                        break;
+                }
+        }
+        srand(getpid());
+
+        /*
+         * We need three parameters
+         */
+        if (argc - optind < 3)
+                usage("Arguments missing.");
+
+        wcet_ms   = atof(argv[optind + 0]);
+        period_ms = atof(argv[optind + 1]);
+
+        wcet   = ms2ns(wcet_ms);
+        period = ms2ns(period_ms);
+        budget = ms2ns(budget_ms);
+        if (wcet <= 0)
+                usage("The worst-case execution time must be a "
+                                "positive number.");
+        if (period <= 0)
+                usage("The period must be a positive number.");
+        if (wcet > period) {
+                usage("The worst-case execution time must not "
+                                "exceed the period.");
+        }
+
+        duration  = atof(argv[optind + 2]);
+        
+        if (migrate) {
+                ret = be_migrate_to_domain(cluster);
+                if (ret < 0)
+                        bail_out("could not migrate to target partition or cluster.");
+        }
+
+        /* reservation config */
+        config.id = gettid();
+        config.polling_params.budget = budget;
+        config.polling_params.period = period;
+        config.polling_params.offset = 0;
+        config.polling_params.relative_deadline = 0;
+        
+        if (config.polling_params.budget > config.polling_params.period) {
+                usage("The budget must not exceed the period.");
+        }
+        
+        /* create a reservation */
+        ret = reservation_create(res_type, &config);
+        if (ret < 0) {
+                bail_out("failed to create reservation.");
+        }
+        
+        init_rt_task_param(&param);
+        param.exec_cost = wcet;
+        param.period = period;
+        param.priority = priority;
+        param.cls = RT_CLASS_HARD;
+        param.release_policy = TASK_PERIODIC;
+        param.budget_policy = NO_ENFORCEMENT;
+        if (migrate) {
+                param.cpu = gettid();
+        }
+        ret = set_rt_task_param(gettid(), &param);
+        if (ret < 0)
+                bail_out("could not setup rt task params");
+        
+        mc2_param.res_id = gettid();
+        ret = set_mc2_task_param(gettid(), &mc2_param);
+        if (ret < 0)
+                bail_out("could not setup mc2 task params");
+        
+        
+        arena_sz = wss*1024;
+        
+        arena = alloc_shm(cs, arena_sz, 0, 0);
+        if (!arena)
+                bail_out("alloc_shm failed.\n");
+        init_arena(arena, arena_sz);
+        
+        local_buf = alloc_local(arena_sz, 0, 0);
+        if (!local_buf)
+                bail_out("alloc_local failed.\n");
+        init_arena(local_buf, arena_sz);
+        
+        ret = init_litmus();
+        if (ret != 0)
+                bail_out("init_litmus() failed\n");
+
+        start = wctime();
+        ret = task_mode(LITMUS_RT_TASK);
+        if (ret != 0)
+                bail_out("could not become RT task");
+
+        if (mc2_param.crit == CRIT_LEVEL_C)
+                set_page_color(-1);
+        else
+                set_page_color(config.cpu);
+
+        mlockall(MCL_CURRENT | MCL_FUTURE);
+//      init_arena(local_buf, arena_sz);
+//      test_call(1);
+//      init_arena(arena, arena_sz);
+//      test_call(2);
+        
+        if (wait) {
+                ret = wait_for_ts_release();
+                if (ret != 0)
+                        bail_out("wait_for_ts_release()");
+                start = wctime();
+        }
+
+        //while (job(wss, wcet_ms * 0.001, start + duration)) {};
+        job(cs, wss, wcet_ms * 0.001, start + duration);
+
+        ret = task_mode(BACKGROUND_TASK);
+        if (ret != 0)
+                bail_out("could not become regular task (huh?)");
+
+        reservation_destroy(gettid(), config.cpu);
+        dealloc_arena(arena, arena_sz);
+        dealloc_arena(local_buf, arena_sz);
+        printf("%s finished.\n", argv[0]);
+        for (i = 0; i<loops; i++)
+                printf("%llu\n", exectime[i]);
+        return 0;
+}
diff --git a/bin/mc2shm.c b/bin/mc2shm.c
new file mode 100644
index 0000000..4cc974a
--- /dev/null
+++ b/bin/mc2shm.c
@@ -0,0 +1,502 @@
+#include <sys/time.h>
+#include <sys/mman.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <string.h>
+#include <assert.h>
+#include <limits.h>
+#include <fcntl.h>
+
+#include "litmus.h"
+#include "common.h"
+#include "color_shm.h"
+
+#define PAGE_SIZE (4096)
+#define CACHELINE_SIZE 32
+#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int))
+#define CACHELINES_IN_1KB (1024 / sizeof(cacheline_t))
+#define INTS_IN_1KB     (1024 / sizeof(int))
+#define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int))
+
+//typedef struct cacheline
+//{
+//        int line[INTS_IN_CACHELINE];
+//} __attribute__((aligned(CACHELINE_SIZE))) cacheline_t;
+
+static int loops = 10;
+static cacheline_t* arena = NULL;
+static cacheline_t* local_buf = NULL;
+
+struct timeval t1,t2;
+
+inline unsigned long get_cyclecount (void)
+{
+        unsigned long value;
+        // Read CCNT Register
+        asm volatile ("MRC p15, 0, %0, c9, c13, 0\t\n": "=r"(value));
+        return value;
+}
+
+#define UNCACHE_DEV "/dev/litmus/uncache"
+
+static cacheline_t* alloc_shm(size_t size, int use_huge_pages, int use_uncache_pages)
+{
+        int ret, fd;
+        struct color_ioctl_cmd shm_info;
+        cacheline_t* arena = NULL;
+        
+        /* allocate shm */
+        fd = open("/dev/litmus/color_shm", O_RDWR);
+        if (fd < 0) {
+                printf("Device open error.\n");
+                return NULL;
+        }
+
+        shm_info.color = 0x00000f00; 
+        //shm_info.bank  = 0x00000020; // hi crit. bank
+        shm_info.bank  = 0x00000040; // levelC
+
+        ret = ioctl(fd, SET_COLOR_SHM_CMD, &shm_info);
+        if (ret < 0) {
+                printf("ioctl failed.\n");
+                return NULL;
+        }
+                
+        arena = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+        if (arena == MAP_FAILED) {
+                printf("mmap failed.\n");
+                return NULL;
+        }
+        close(fd);
+
+        mlockall(MCL_CURRENT | MCL_FUTURE);
+        /* finish allocation */
+
+    assert(arena);
+
+    return arena;
+}
+
+static cacheline_t* alloc_local(size_t size, int use_huge_pages, int use_uncache_pages)
+{
+    int flags = MAP_PRIVATE | MAP_POPULATE;
+    cacheline_t* arena = NULL;
+    int fd;
+
+    if(use_huge_pages)
+        flags |= MAP_HUGETLB;
+
+        if(use_uncache_pages) {
+                        fd = open(UNCACHE_DEV, O_RDWR|O_SYNC);
+                        if (fd == -1)
+                                        bail_out("Failed to open uncache device. Are you running the LITMUS^RT kernel?");
+        }
+        else {
+                        fd = -1;
+                        flags |= MAP_ANONYMOUS;
+        }
+
+    arena = mmap(0, size, PROT_READ | PROT_WRITE, flags, fd, 0);
+        
+    if(use_uncache_pages)
+                close(fd);
+
+    assert(arena);
+
+    return arena;
+}
+
+static void dealloc_arena(cacheline_t* arena, size_t size)
+{
+                int ret = munmap((void*)arena, size);
+        if(ret != 0)
+                bail_out("munmap() error");
+}
+
+static int randrange(int min, int max)
+{
+        /* generate a random number on the range [min, max) w/o skew */
+        int limit = max - min;
+        int devisor = RAND_MAX/limit;
+        int retval;
+
+        do {
+                retval = rand() / devisor;
+        } while(retval == limit);
+        retval += min;
+
+        return retval;
+}
+
+static void init_arena(cacheline_t* arena, size_t size)
+{
+    int i;
+        size_t num_arena_elem = size / sizeof(cacheline_t);
+
+        /* Generate a cycle among the cache lines using Sattolo's algorithm.
+           Every int in the cache line points to the same cache line.
+           Note: Sequential walk doesn't care about these values. */
+        for (i = 0; i < num_arena_elem; i++) {
+                int j;
+                for(j = 0; j < INTS_IN_CACHELINE; ++j)
+                        arena[i].line[j] = i;
+        }
+        while(1 < i--) {
+                int j = randrange(0, i);
+                cacheline_t temp = arena[j];
+                arena[j] = arena[i];
+                arena[i] = temp;
+        }
+}
+
+/* Random walk around the arena in cacheline-sized chunks.
+   Cacheline-sized chucks ensures the same utilization of each
+   hit line as sequential read. (Otherwise, our utilization
+   would only be 1/INTS_IN_CACHELINE.) */
+static int random_walk(cacheline_t *mem, int wss, int write_cycle)
+{
+        /* a random cycle among the cache lines was set up by init_arena(). */
+        int sum, i, next, j;
+
+        int numlines = wss * CACHELINES_IN_1KB;
+
+        sum = 0;
+
+        /* contents of arena is structured s.t. offsets are all
+           w.r.t. to start of arena, so compute the initial offset */
+        next = mem - arena;
+
+        if (write_cycle == 0) {
+                for (i = 0; i < numlines; i++) {
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                /* every element in the cacheline has the same value */
+                                //next = arena[next].line[j];
+                                local_buf[next].line[j] = arena[next].line[j];
+                                //sum += next;
+                                //local_buf[next].line[j] = next;
+                        }
+                }
+        }
+        
+        else {
+                int w, which_line;
+                for (i = 0, w = 0; i < numlines; i++) {
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                which_line = next;
+                                next = local_buf[next].line[j];
+                                ((volatile cacheline_t*)arena)[which_line].line[j] = next;
+                                //sum += next;
+                        }
+                }
+        }
+        return sum;
+}
+
+static cacheline_t* random_start(int wss)
+{
+        return arena + randrange(0, ((wss * 1024)/sizeof(cacheline_t)));
+}
+
+static int sequential_walk(cacheline_t *mem, int wss, int write_cycle)
+{
+    int sum = 0, i, j;
+    //int* mem = (int*)_mem; /* treat as raw buffer of ints */
+    int num_ints = wss * CACHELINES_IN_1KB;
+
+    if (write_cycle > 0) {
+        for (i = 0; i < num_ints; i++) {
+            //if (i % write_cycle == (write_cycle - 1)) {
+                                //mem[i]++;
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                //sum += local_buf[i].line[j];
+                                mem[i].line[j] = local_buf[i].line[j];;
+                        }
+                        //}
+            //else {
+            //    sum += mem[i];
+                        //}
+        }
+    } else {
+        /* sequential access, pure read */
+        for (i = 0; i < num_ints; i++) {
+                        for (j = 0; j < INTS_IN_CACHELINE; j++) {
+                                //sum += mem[i];
+                                local_buf[i].line[j] = mem[i].line[j];
+                                //sum += mem[i];
+                        }
+                }
+    }
+    return sum;
+}
+
+static cacheline_t* sequential_start(int wss)
+{
+        return arena;
+/*    static int pos = 0;
+
+    int num_cachelines = wss * CACHELINES_IN_1KB;
+        size_t num_arena_elem = wss * 1024 / sizeof(cacheline_t);
+    cacheline_t *mem;
+*/      
+        
+    /* Don't allow re-use between allocations.
+     * At most half of the arena may be used
+     * at any one time.
+     */
+/*    if (num_cachelines * 2 > num_arena_elem)
+        bail_out("static memory arena too small");
+
+    if (pos + num_cachelines > num_arena_elem) {
+        mem = arena;
+        pos = num_cachelines;
+    } else {
+        mem = arena + pos;
+        pos += num_cachelines;
+    }
+
+    return mem;*/
+}
+
+static volatile int dont_optimize_me = 0;
+
+static void usage(char *error) {
+        fprintf(stderr, "Error: %s\n", error);
+        fprintf(stderr,
+                "Usage:\n"
+                "       rt_spin [COMMON-OPTS] WCET PERIOD DURATION\n"
+                "       rt_spin [COMMON-OPTS] -f FILE [-o COLUMN] WCET PERIOD\n"
+                "       rt_spin -l\n"
+                "\n"
+                "COMMON-OPTS = [-w] [-s SCALE]\n"
+                "              [-p PARTITION/CLUSTER [-z CLUSTER SIZE]] [-m CRITICALITY LEVEL]\n"
+                "              [-k WSS] [-l LOOPS] [-b BUDGET]\n"
+                "\n"
+                "WCET and PERIOD are milliseconds, DURATION is seconds.\n");
+        exit(EXIT_FAILURE);
+}
+
+static int loop_once(int wss)
+{
+        cacheline_t *mem;
+        int temp;
+        
+        mem = random_start(wss);
+        temp = random_walk(mem, wss, 1);
+    
+        //mem = sequential_start(wss);
+        //temp = sequential_walk(arena, wss, 1);
+        dont_optimize_me = temp;
+        
+        return dont_optimize_me;
+}
+
+static int job(int wss, double exec_time, double program_end)
+{
+        if (wctime() > program_end)
+                return 0;
+        else {
+                register unsigned long t;
+                register unsigned int iter = 0;
+                //t = get_cyclecount();
+                //gettimeofday(&t1, NULL);
+                while(iter++ < loops) {
+                        loop_once(wss);
+                }
+                //gettimeofday(&t2, NULL);
+                //printf("%ld cycles\n", get_cyclecount() - t);
+                //printf("%ld\n", ((t2.tv_sec * 1000000 + t2.tv_usec) - (t1.tv_sec * 1000000 + t1.tv_usec)));
+                sleep_next_period(); 
+        return 1;
+        }
+}
+
+#define OPTSTR "p:wl:m:i:b:k:"
+int main(int argc, char** argv)
+{
+        int ret;
+        lt_t wcet, period, budget;
+        double wcet_ms, period_ms, budget_ms;
+        unsigned int priority = LITMUS_NO_PRIORITY;
+        int migrate = 0;
+        int cluster = 0;
+        int opt;
+        int wait = 0;
+        double duration = 0, start = 0;
+        struct rt_task param;
+        struct mc2_task mc2_param;
+        struct reservation_config config;
+        int res_type = PERIODIC_POLLING;
+        size_t arena_sz;
+        int wss;
+
+        /* default for reservation */
+        config.id = 0;
+        config.priority = LITMUS_NO_PRIORITY; /* use EDF by default */
+        config.cpu = -1;
+        
+        mc2_param.crit = CRIT_LEVEL_C;
+        
+        budget_ms = 1000;
+        
+        while ((opt = getopt(argc, argv, OPTSTR)) != -1) {
+                switch (opt) {
+                case 'w':
+                        wait = 1;
+                        break;
+                case 'p':
+                        cluster = atoi(optarg);
+                        migrate = 1;
+                        config.cpu = cluster;
+                        break;
+                case 'l':
+                        loops = atoi(optarg);
+                        break;
+                case 'k':
+                        wss = atoi(optarg);
+                        break;
+                case 'm':
+                        mc2_param.crit = atoi(optarg);
+                        if ((mc2_param.crit >= CRIT_LEVEL_A) && (mc2_param.crit <= CRIT_LEVEL_C)) {
+                                res_type = PERIODIC_POLLING;
+                        }
+                        else
+                                usage("Invalid criticality level.");
+                        break;
+                case 'b':
+                        budget_ms = atof(optarg);
+                        break;
+                case 'i':
+                        config.priority = atoi(optarg);
+                        break;
+                case ':':
+                        usage("Argument missing.");
+                        break;
+                case '?':
+                default:
+                        usage("Bad argument.");
+                        break;
+                }
+        }
+        srand(getpid());
+
+        /*
+         * We need three parameters
+         */
+        if (argc - optind < 3)
+                usage("Arguments missing.");
+
+        wcet_ms   = atof(argv[optind + 0]);
+        period_ms = atof(argv[optind + 1]);
+
+        wcet   = ms2ns(wcet_ms);
+        period = ms2ns(period_ms);
+        budget = ms2ns(budget_ms);
+        if (wcet <= 0)
+                usage("The worst-case execution time must be a "
+                                "positive number.");
+        if (period <= 0)
+                usage("The period must be a positive number.");
+        if (wcet > period) {
+                usage("The worst-case execution time must not "
+                                "exceed the period.");
+        }
+
+        duration  = atof(argv[optind + 2]);
+        
+        if (migrate) {
+                ret = be_migrate_to_domain(cluster);
+                if (ret < 0)
+                        bail_out("could not migrate to target partition or cluster.");
+        }
+
+        /* reservation config */
+        config.id = gettid();
+        config.polling_params.budget = budget;
+        config.polling_params.period = period;
+        config.polling_params.offset = 0;
+        config.polling_params.relative_deadline = 0;
+        
+        if (config.polling_params.budget > config.polling_params.period) {
+                usage("The budget must not exceed the period.");
+        }
+        
+        /* create a reservation */
+        ret = reservation_create(res_type, &config);
+        if (ret < 0) {
+                bail_out("failed to create reservation.");
+        }
+        
+        init_rt_task_param(&param);
+        param.exec_cost = wcet;
+        param.period = period;
+        param.priority = priority;
+        param.cls = RT_CLASS_HARD;
+        param.release_policy = TASK_PERIODIC;
+        param.budget_policy = NO_ENFORCEMENT;
+        if (migrate) {
+                param.cpu = gettid();
+        }
+        ret = set_rt_task_param(gettid(), &param);
+        if (ret < 0)
+                bail_out("could not setup rt task params");
+        
+        mc2_param.res_id = gettid();
+        ret = set_mc2_task_param(gettid(), &mc2_param);
+        if (ret < 0)
+                bail_out("could not setup mc2 task params");
+        
+        
+        arena_sz = wss*1024;
+        
+        arena = alloc_shm(arena_sz, 0, 0);
+        if (!arena)
+                bail_out("alloc_shm failed.\n");
+        init_arena(arena, arena_sz);
+        
+        local_buf = alloc_local(arena_sz, 0, 0);
+        if (!local_buf)
+                bail_out("alloc_local failed.\n");
+        init_arena(local_buf, arena_sz);
+        
+        ret = init_litmus();
+        if (ret != 0)
+                bail_out("init_litmus() failed\n");
+
+        start = wctime();
+        ret = task_mode(LITMUS_RT_TASK);
+        if (ret != 0)
+                bail_out("could not become RT task");
+
+        if (mc2_param.crit == CRIT_LEVEL_C)
+                set_page_color(-1);
+        else
+                set_page_color(config.cpu);
+
+        mlockall(MCL_CURRENT | MCL_FUTURE);
+//      init_arena(local_buf, arena_sz);
+//      test_call(1);
+//      init_arena(arena, arena_sz);
+//      test_call(2);
+        
+        if (wait) {
+                ret = wait_for_ts_release();
+                if (ret != 0)
+                        bail_out("wait_for_ts_release()");
+                start = wctime();
+        }
+
+        while (job(wss, wcet_ms * 0.001, start + duration)) {};
+
+        ret = task_mode(BACKGROUND_TASK);
+        if (ret != 0)
+                bail_out("could not become regular task (huh?)");
+
+        reservation_destroy(gettid(), config.cpu);
+        dealloc_arena(arena, arena_sz);
+        dealloc_arena(local_buf, arena_sz);
+        printf("%s finished.\n", argv[0]);
+        return 0;
+}
diff --git a/bin/mttest.c b/bin/mtdag.c
index d502778..adf6d8d 100644
--- a/bin/mttest.c
+++ b/bin/mtdag.c
@@ -26,6 +26,7 @@
 /* Include the LITMUS^RT API.*/
 #include "litmus.h"
 #include "color_shm.h"
+#include "asm/cycles.h"
 
 #define PERIOD            100
 #define RELATIVE_DEADLINE 100
@@ -41,7 +42,9 @@ struct thread_context {
         int id;
         int cpu;
         int job_no;
-        char* shm;
+        char* shm1;
+        char* shm2;
+        char* shm3;
 };
 
 /* The real-time thread program. Doesn't have to be the same for
@@ -73,11 +76,14 @@ int job(struct thread_context *tcx);
  */
 int main(int argc, char** argv)
 {
-        int i, ret, fd;
+        int i;
         struct thread_context ctx[NUM_THREADS];
         pthread_t             task[NUM_THREADS];
-        char *shm;
+        char *shm1;
+        char *shm2;
+        char *shm3;
         struct color_ioctl_cmd shm_info;
+        struct color_ioctl_offset shm_offset;
         /* The task is in background mode upon startup. */              
 
 
@@ -91,27 +97,31 @@ int main(int argc, char** argv)
          * 2) Work environment (e.g., global data structures, file data, etc.) would
          *    be setup here.
          */
-        fd = open("/dev/litmus/color_shm", O_RDWR);
-        if (fd < 0) {
-                printf("Device open error.\n");
-                return -1;
-        }
-
-        shm_info.color = 0x0000cccc;
-        shm_info.bank = 0x000000ac;
 
-        ret = ioctl(fd, SET_COLOR_SHM_CMD, &shm_info);
-        if (ret < 0) {
-                printf("ioctl failed.\n");
-                return -1;
+        shm_info.color = 0x00000001;
+        shm_info.bank = 0x00000020;
+        shm_offset.offset = 0;
+        shm_offset.lock = 1;
+        
+        shm1 = (char*)color_mmap(1024, shm_info, shm_offset);
+        if (!shm1) {
+                printf("color mmap failed.\n");
+                exit(-1);
         }
-                
-        shm = mmap(NULL, 6000, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-        if (shm == MAP_FAILED) {
-                printf("mmap failed.\n");
-                return -1;
+        else {
+                printf("Mapped vaddr = %p\n", shm1);
+        }
+
+        shm_info.color = 0x00000003;
+        shm_offset.offset = 1024;
+        shm2 = (char*)color_mmap(4096, shm_info, shm_offset);
+        if (!shm2) {
+                printf("color mmap failed.\n");
+                exit(-1);
+        }
+        else {
+                printf("Mapped vaddr = %p\n", shm2);
         }
-        close(fd);
 
         mlockall(MCL_CURRENT | MCL_FUTURE);
         /*****
@@ -128,7 +138,9 @@ int main(int argc, char** argv)
                 ctx[i].id = i;
                 ctx[i].cpu = 0;
                 ctx[i].job_no = 0;
-                ctx[i].shm = shm;
+                ctx[i].shm1 = shm1;
+                ctx[i].shm3 = shm2;
+                ctx[i].shm2 = shm3;
                 pthread_create(task + i, NULL, rt_thread, (void *) (ctx + i));
         }
 
@@ -253,24 +265,22 @@ void* rt_thread(void *tcontext)
 int job(struct thread_context *tcx) 
 {
         int i;
-        char* buf = tcx->shm;
+        char* buf1 = tcx->shm1;
+        char* buf2 = tcx->shm2;
+        char* buf3 = tcx->shm3;
+        char tmp = 0;
         /* Do real-time calculation. */
         printf("Task %d Job %d executinig\n", tcx->id, tcx->job_no);
 
         if (tcx->id == 0) {
-                printf("WRITE\n");
-                for (i=0; i<600; i++) {
-                        buf[i] = rand()%255;
-                        printf("%x ",buf[i]);
-                }
-                printf("\n");
-        } else if (tcx->id == 1) {
                 printf("READ\n");
                 for (i=0; i<600; i++) {
-                        char t = buf[i];
+                        char t = buf1[i];
                         printf("%x ", t);
                 }
                 printf("\n");
+
+                ;
         }
         //test_call(0);
         tcx->job_no++;
diff --git a/bin/test1.c b/bin/test1.c
new file mode 100644
index 0000000..a38d402
--- /dev/null
+++ b/bin/test1.c
@@ -0,0 +1,453 @@
+#include <sys/time.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+#include <string.h>
+#include <assert.h>
+#include <limits.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "litmus.h"
+#include "common.h"
+
+
+
+static void usage(char *error) {
+        fprintf(stderr, "Error: %s\n", error);
+        fprintf(stderr,
+                "Usage:\n"
+                "       rt_spin [COMMON-OPTS] WCET PERIOD DURATION\n"
+                "       rt_spin [COMMON-OPTS] -f FILE [-o COLUMN] WCET PERIOD\n"
+                "       rt_spin -l\n"
+                "\n"
+                "COMMON-OPTS = [-w] [-s SCALE]\n"
+                "              [-p PARTITION/CLUSTER [-z CLUSTER SIZE]] [-c CLASS]\n"
+                "              [-X LOCKING-PROTOCOL] [-L CRITICAL SECTION LENGTH] [-Q RESOURCE-ID]"
+                "\n"
+                "WCET and PERIOD are milliseconds, DURATION is seconds.\n"
+                "CRITICAL SECTION LENGTH is in milliseconds.\n");
+        exit(EXIT_FAILURE);
+}
+
+/*
+ * returns the character that made processing stop, newline or EOF
+ */
+static int skip_to_next_line(FILE *fstream)
+{
+        int ch;
+        for (ch = fgetc(fstream); ch != EOF && ch != '\n'; ch = fgetc(fstream));
+        return ch;
+}
+
+static void skip_comments(FILE *fstream)
+{
+        int ch;
+        for (ch = fgetc(fstream); ch == '#'; ch = fgetc(fstream))
+                skip_to_next_line(fstream);
+        ungetc(ch, fstream);
+}
+
+static void get_exec_times(const char *file, const int column,
+                           int *num_jobs,    double **exec_times)
+{
+        FILE *fstream;
+        int  cur_job, cur_col, ch;
+        *num_jobs = 0;
+
+        fstream = fopen(file, "r");
+        if (!fstream)
+                bail_out("could not open execution time file");
+
+        /* figure out the number of jobs */
+        do {
+                skip_comments(fstream);
+                ch = skip_to_next_line(fstream);
+                if (ch != EOF)
+                        ++(*num_jobs);
+        } while (ch != EOF);
+
+        if (-1 == fseek(fstream, 0L, SEEK_SET))
+                bail_out("rewinding file failed");
+
+        /* allocate space for exec times */
+        *exec_times = calloc(*num_jobs, sizeof(*exec_times));
+        if (!*exec_times)
+                bail_out("couldn't allocate memory");
+
+        for (cur_job = 0; cur_job < *num_jobs && !feof(fstream); ++cur_job) {
+
+                skip_comments(fstream);
+
+                for (cur_col = 1; cur_col < column; ++cur_col) {
+                        /* discard input until we get to the column we want */
+                        int unused __attribute__ ((unused)) = fscanf(fstream, "%*s,");
+                }
+
+                /* get the desired exec. time */
+                if (1 != fscanf(fstream, "%lf", (*exec_times)+cur_job)) {
+                        fprintf(stderr, "invalid execution time near line %d\n",
+                                        cur_job);
+                        exit(EXIT_FAILURE);
+                }
+
+                skip_to_next_line(fstream);
+        }
+
+        assert(cur_job == *num_jobs);
+        fclose(fstream);
+}
+
+#define NUMS 4096
+static int num[NUMS];
+static char* progname;
+static char* smem;
+static int access_type; /* 0: read 1: write */
+
+static int loop_once(void)
+{
+        int i, j = 0;
+        for (i = 0; i < NUMS; i++) {
+                j += num[i]++;
+                if (access_type == 1) {
+                        smem[i] = j%255;
+                        printf("Write %d at [%d]\n", smem[i], i);
+                } else {
+                        char tmp = smem[i];
+                        printf("Read %d at [%d]\n", tmp, i);
+                }
+        }
+        return j;
+}
+
+static int loop_for(double exec_time, double emergency_exit)
+{
+        double last_loop = 0, loop_start;
+        int tmp = 0;
+
+        double start = cputime();
+        double now = cputime();
+
+        while (now + last_loop < start + exec_time) {
+                loop_start = now;
+                tmp += loop_once();
+                now = cputime();
+                last_loop = now - loop_start;
+                if (emergency_exit && wctime() > emergency_exit) {
+                        /* Oops --- this should only be possible if the execution time tracking
+                         * is broken in the LITMUS^RT kernel. */
+                        fprintf(stderr, "!!! rtspin/%d emergency exit!\n", getpid());
+                        fprintf(stderr, "Something is seriously wrong! Do not ignore this.\n");
+                        break;
+                }
+        }
+
+        return tmp;
+}
+
+
+static void debug_delay_loop(void)
+{
+        double start, end, delay;
+
+        while (1) {
+                for (delay = 0.5; delay > 0.01; delay -= 0.01) {
+                        start = wctime();
+                        loop_for(delay, 0);
+                        end = wctime();
+                        printf("%6.4fs: looped for %10.8fs, delta=%11.8fs, error=%7.4f%%\n",
+                               delay,
+                               end - start,
+                               end - start - delay,
+                               100 * (end - start - delay) / delay);
+                }
+        }
+}
+
+static int job(double exec_time, double program_end, int lock_od, double cs_length)
+{
+        double chunk1, chunk2;
+
+        if (wctime() > program_end)
+                return 0;
+        else {
+                if (lock_od >= 0) {
+                        /* simulate critical section somewhere in the middle */
+                        chunk1 = drand48() * (exec_time - cs_length);
+                        chunk2 = exec_time - cs_length - chunk1;
+
+                        /* non-critical section */
+                        loop_for(chunk1, program_end + 1);
+
+                        /* critical section */
+                        litmus_lock(lock_od);
+                        loop_for(cs_length, program_end + 1);
+                        litmus_unlock(lock_od);
+
+                        /* non-critical section */
+                        loop_for(chunk2, program_end + 2);
+                } else {
+                        //loop_for(exec_time, program_end + 1);
+                        loop_once();
+                }
+                sleep_next_period();
+                return 1;
+        }
+}
+
+#define OPTSTR "p:c:wlveo:f:s:q:r:X:L:Q:vt:"
+int main(int argc, char** argv)
+{
+        int ret;
+        lt_t wcet;
+        lt_t period;
+        double wcet_ms, period_ms;
+        unsigned int priority = LITMUS_NO_PRIORITY;
+        int migrate = 0;
+        int cluster = 0;
+        int reservation = -1;
+        int opt;
+        int wait = 0;
+        int test_loop = 0;
+        int column = 1;
+        const char *file = NULL;
+        int want_enforcement = 0;
+        double duration = 0, start = 0;
+        double *exec_times = NULL;
+        double scale = 1.0;
+        task_class_t class = RT_CLASS_HARD;
+        int cur_job = 0, num_jobs = 0;
+        struct rt_task param;
+
+        int verbose = 0;
+        unsigned int job_no;
+
+        /* locking */
+        int lock_od = -1;
+        int resource_id = 0;
+        const char *lock_namespace = "./rtspin-locks";
+        int protocol = -1;
+        double cs_length = 1; /* millisecond */
+        int fd;
+
+        progname = argv[0];
+
+        while ((opt = getopt(argc, argv, OPTSTR)) != -1) {
+                switch (opt) {
+                case 'w':
+                        wait = 1;
+                        break;
+                case 'p':
+                        cluster = atoi(optarg);
+                        migrate = 1;
+                        break;
+                case 'r':
+                        reservation = atoi(optarg);
+                        break;
+                case 'q':
+                        priority = atoi(optarg);
+                        if (!litmus_is_valid_fixed_prio(priority))
+                                usage("Invalid priority.");
+                        break;
+                case 'c':
+                        class = str2class(optarg);
+                        if (class == -1)
+                                usage("Unknown task class.");
+                        break;
+                case 'e':
+                        want_enforcement = 1;
+                        break;
+                case 'l':
+                        test_loop = 1;
+                        break;
+                case 'o':
+                        column = atoi(optarg);
+                        break;
+                case 'f':
+                        file = optarg;
+                        break;
+                case 's':
+                        scale = atof(optarg);
+                        break;
+                case 'X':
+                        protocol = lock_protocol_for_name(optarg);
+                        if (protocol < 0)
+                                usage("Unknown locking protocol specified.");
+                        break;
+                case 'L':
+                        cs_length = atof(optarg);
+                        if (cs_length <= 0)
+                                usage("Invalid critical section length.");
+                        break;
+                case 'Q':
+                        resource_id = atoi(optarg);
+                        if (resource_id <= 0 && strcmp(optarg, "0"))
+                                usage("Invalid resource ID.");
+                        break;
+                case 'v':
+                        verbose = 1;
+                        break;
+                case 't':
+                        access_type = atoi(optarg);
+                        if (access_type != 0 && access_type != 1)
+                                usage("Wrong SHM access type.");
+                        break;
+                case ':':
+                        usage("Argument missing.");
+                        break;
+                case '?':
+                default:
+                        usage("Bad argument.");
+                        break;
+                }
+        }
+
+        if (test_loop) {
+                debug_delay_loop();
+                return 0;
+        }
+
+        srand(getpid());
+
+        if (file) {
+                get_exec_times(file, column, &num_jobs, &exec_times);
+
+                if (argc - optind < 2)
+                        usage("Arguments missing.");
+
+                for (cur_job = 0; cur_job < num_jobs; ++cur_job) {
+                        /* convert the execution time to seconds */
+                        duration += exec_times[cur_job] * 0.001;
+                }
+        } else {
+                /*
+                 * if we're not reading from the CSV file, then we need
+                 * three parameters
+                 */
+                if (argc - optind < 3)
+                        usage("Arguments missing.");
+        }
+
+        wcet_ms   = atof(argv[optind + 0]);
+        period_ms = atof(argv[optind + 1]);
+
+        wcet   = ms2ns(wcet_ms);
+        period = ms2ns(period_ms);
+        if (wcet <= 0)
+                usage("The worst-case execution time must be a "
+                                "positive number.");
+        if (period <= 0)
+                usage("The period must be a positive number.");
+        if (!file && wcet > period) {
+                usage("The worst-case execution time must not "
+                                "exceed the period.");
+        }
+
+        if (!file)
+                duration  = atof(argv[optind + 2]);
+        else if (file && num_jobs > 1)
+                duration += period_ms * 0.001 * (num_jobs - 1);
+
+        if (migrate) {
+                ret = be_migrate_to_domain(cluster);
+                if (ret < 0)
+                        bail_out("could not migrate to target partition or cluster.");
+        }
+
+
+        /* allocate shared memory */
+        fd = open("/dev/shm/region1", O_CREAT|O_RDWR|O_TRUNC, 0666);
+        if (fd == -1)
+                bail_out("could not open /dev/shm.");
+        
+        fallocate(fd, 0, 0, 4096);
+        smem = NULL;
+        if (access_type == 1) {
+                printf("open write mmap now.\n");
+                smem = (char*) mmap(NULL, 4096, PROT_WRITE|PROT_READ, MAP_SHARED, fd, 0);
+                if (smem == MAP_FAILED)
+                        bail_out("mmap error.");
+                close(fd);
+        }
+        else {
+                printf("open read mmap now.\n");
+                smem = (char*) mmap(NULL, 4096, PROT_WRITE|PROT_READ, MAP_SHARED, fd, 0);
+                if (smem == MAP_FAILED)
+                        bail_out("mmap error.");
+                close(fd);
+        }
+        printf("smem addr = %p\n", smem);
+        init_rt_task_param(&param);
+        param.exec_cost = wcet;
+        param.period = period;
+        param.priority = priority;
+        param.cls = class;
+        param.budget_policy = (want_enforcement) ?
+                        PRECISE_ENFORCEMENT : NO_ENFORCEMENT;
+        if (migrate) {
+                if (reservation >= 0)
+                        param.cpu = reservation;
+                else
+                        param.cpu = domain_to_first_cpu(cluster);
+        }
+        ret = set_rt_task_param(gettid(), &param);
+        if (ret < 0)
+                bail_out("could not setup rt task params");
+
+        init_litmus();
+//test_call(0);
+        start = wctime();
+        ret = task_mode(LITMUS_RT_TASK);
+        if (ret != 0)
+                bail_out("could not become RT task");
+
+        if (protocol >= 0) {
+                /* open reference to semaphore */
+                lock_od = litmus_open_lock(protocol, resource_id, lock_namespace, &cluster);
+                if (lock_od < 0) {
+                        perror("litmus_open_lock");
+                        usage("Could not open lock.");
+                }
+        }
+
+
+        if (wait) {
+                ret = wait_for_ts_release();
+                if (ret != 0)
+                        bail_out("wait_for_ts_release()");
+                start = wctime();
+        }
+
+        if (file) {
+                /* use times read from the CSV file */
+                for (cur_job = 0; cur_job < num_jobs; ++cur_job) {
+                        /* convert job's length to seconds */
+                        job(exec_times[cur_job] * 0.001 * scale,
+                            start + duration,
+                            lock_od, cs_length * 0.001);
+                }
+        } else {
+                do {
+                        if (verbose) {
+                                get_job_no(&job_no);
+                                printf("rtspin/%d:%u @ %.4fms\n", gettid(),
+                                        job_no, (wctime() - start) * 1000);
+                        }
+                        /* convert to seconds and scale */
+                } while (job(wcet_ms * 0.001 * scale, start + duration,
+                           lock_od, cs_length * 0.001));
+        }
+
+        ret = task_mode(BACKGROUND_TASK);
+        if (ret != 0)
+                bail_out("could not become regular task (huh?)");
+
+        if (file)
+                free(exec_times);
+
+        return 0;
+}
diff --git a/include/color_shm.h b/include/color_shm.h
index ffa90dd..449ec9d 100644
--- a/include/color_shm.h
+++ b/include/color_shm.h
@@ -18,4 +18,3 @@ struct color_ioctl_offset {
 #define SET_COLOR_SHM_OFFSET    _IOW(SHM_MAJOR, 0x2, struct color_ioctl_offset)
 
 void* color_mmap(size_t size, struct color_ioctl_cmd cmd, struct color_ioctl_offset offset);
-
diff --git a/include/litmus.h b/include/litmus.h
index 4a4ea39..7229a38 100644
--- a/include/litmus.h
+++ b/include/litmus.h
@@ -46,7 +46,7 @@ extern "C" {
  * @private
  * Number of semaphore protocol object types
  */
-#define SCHED_LITMUS 7
+#define SCHED_LITMUS 6
 
 #define CACHELINE_SIZE 32
 #define INTS_IN_CACHELINE (CACHELINE_SIZE/sizeof(int))
diff --git a/src/kernel_iface.c b/src/kernel_iface.c
index 49cef2a..a2c2104 100644
--- a/src/kernel_iface.c
+++ b/src/kernel_iface.c
@@ -140,62 +140,6 @@ int requested_to_preempt(void)
         return (likely(ctrl_page != NULL) && ctrl_page->sched.np.preempt);
 }
 
-void enter_pgm_wait(void)
-{
-        if (likely(ctrl_page != NULL) || init_kernel_iface() == 0) {
-                assert(!ctrl_page->pgm_waiting);
-                ctrl_page->pgm_waiting = 1;
-                __sync_synchronize();
-        }
-        else {
-                fprintf(stderr, "enter_pgm_wait: control page not mapped!\n");
-        }
-}
-
-void exit_pgm_wait(void)
-{
-        if (likely(ctrl_page != NULL)) {
-                assert(ctrl_page->pgm_waiting);
-                ctrl_page->pgm_waiting = 0;
-                __sync_synchronize();
-        }
-        else {
-                fprintf(stderr, "exit_pgm_wait: control page not mapped!\n");
-        }
-}
-
-void enter_pgm_send(void)
-{
-        if (likely(ctrl_page != NULL) || init_kernel_iface() == 0) {
-                assert(!ctrl_page->pgm_sending);
-                ctrl_page->pgm_sending = 1; /* we will become boosted if
-                                               anyone tries to preempt us. */
-                __sync_synchronize();
-        }
-        else {
-                fprintf(stderr, "enter_pgm_send: control page not mapped!\n");
-        }
-}
-
-void exit_pgm_send(void)
-{
-        if (likely(ctrl_page != NULL)) {
-                assert(ctrl_page->pgm_sending);
-
-                ctrl_page->pgm_satisfied = 1;
-                __sync_synchronize();
-
-                /* re-eval priority. Should clear pgm_sending and pgm_satisfied. */
-                sched_yield();
-
-                /* double check that Litmus is doing its job */
-                assert(!ctrl_page->pgm_sending && !ctrl_page->pgm_satisfied);
-        }
-        else {
-                fprintf(stderr, "exit_pgm_send: control page not mapped!\n");
-        }
-}
-
 /* init and return a ptr to the control page for
  * preemption and migration overhead analysis
  *
diff --git a/src/syscalls.c b/src/syscalls.c
index a4cec67..6ec0cd9 100644
--- a/src/syscalls.c
+++ b/src/syscalls.c
@@ -107,11 +107,6 @@ int set_page_color(int cpu)
         return syscall(__NR_set_page_color, cpu);
 }
 
-int test_call(unsigned int param)
-{
-        return syscall(__NR_test_call, param);
-}
-
 int run_bench(int type, int wss, cacheline_t *src, cacheline_t *dst, lt_t *ts)
 {
         return syscall(__NR_run_test, type, wss, src, dst, ts);