Changes to ARM lockdown and LITMUS cycle counting code.

Try and fix the cycle counter on ARMv7 CPUs (unsuccessful).
Use the multi-core timer on the ODROID-X to count cycles.
Add lockdown register access to proc.
Add proc options to disable prefetching for L1/L2 cache.
Add test to lockdown.c to print the time to read in various WSSs.

Signed-off-by: Christopher Kenna <cjk@cs.unc.edu>

11 files changed, 528 insertions, 55 deletions

diff --git a/arch/arm/include/asm/timex.h b/arch/arm/include/asm/timex.h
index 12a53699cd20..7f75e9a0cd32 100644
--- a/arch/arm/include/asm/timex.h
+++ b/arch/arm/include/asm/timex.h
@@ -14,18 +14,7 @@
 
 #include <mach/timex.h>
 
-typedef unsigned long cycles_t;
-
-#if defined(CONFIG_CPU_V7) && !defined(CONFIG_HW_PERF_EVENTS)
-static inline cycles_t v7_get_cycles (void)
-{
-        u32 value;
-        /* read CCNT register */
-        asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(value));
-        return value;
-}
-#define get_cycles v7_get_cycles
-#endif
+typedef u64 cycles_t;
 
 #ifndef get_cycles
 static inline cycles_t get_cycles (void)
diff --git a/include/litmus/clock.h b/include/litmus/clock.h
new file mode 100644
index 000000000000..d6f1cfd2ca60
--- /dev/null
+++ b/include/litmus/clock.h
@@ -0,0 +1,44 @@
+#ifndef _LITMUS_CLOCK_H_
+#define _LITMUS_CLOCK_H_
+
+#if defined(CONFIG_EXYNOS_MCT)
+/*
+ * Only used if we are using the EXYNOS MCT clock.
+ */
+
+#include <linux/clocksource.h>
+extern struct clocksource mct_frc;
+
+static inline cycles_t mct_frc_read(void)
+{
+        cycle_t cycles = mct_frc.read(&mct_frc);
+        return cycles;
+}
+
+static inline s64 litmus_cycles_to_ns(cycles_t cycles)
+{
+        return clocksource_cyc2ns(cycles, mct_frc.mult, mct_frc.shift);
+}
+
+#define litmus_get_cycles mct_frc_read
+
+#elif defined(CONFIG_CPU_V7) && !defined(CONFIG_HW_PERF_EVENTS)
+
+static inline cycles_t v7_get_cycles (void)
+{
+        u32 value;
+        /* read CCNT register */
+        asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(value));
+        return value;
+}
+
+#define litmus_get_cycles v7_get_cycles
+
+#else
+#include <asm/timex.h>
+
+#define litmus_get_cycles get_cycles
+
+#endif
+
+#endif
diff --git a/include/litmus/color.h b/include/litmus/color.h
index b8e8a8d5fce7..465d768b2d6b 100644
--- a/include/litmus/color.h
+++ b/include/litmus/color.h
@@ -20,8 +20,20 @@ struct color_cache_info {
         unsigned long nr_colors;
 };
 
-int litmus_test_prefetch(struct ctl_table *, int, void __user *,
-                size_t *, loff_t *);
+#ifdef CONFIG_CPU_V7
+int litmus_test_prefetch_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
+int litmus_l1_prefetch_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
+int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
+int litmus_l2_double_linefill_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
+int litmus_l2_data_prefetch_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
+int litmus_lockdown_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
+#endif
 
 /* defined in litmus/color.c */
 extern struct color_cache_info color_cache_info;
diff --git a/include/litmus/feather_trace.h b/include/litmus/feather_trace.h
index 028dfb206fb0..a816dc4e6809 100644
--- a/include/litmus/feather_trace.h
+++ b/include/litmus/feather_trace.h
@@ -32,11 +32,11 @@ static inline int fetch_and_dec(int *val)
 
 /* provide default implementation */
 
-#include <asm/timex.h> /* for get_cycles() */
+#include <litmus/clock.h>
 
 static inline unsigned long long ft_timestamp(void)
 {
-        return get_cycles();
+        return litmus_get_cycles();
 }
 
 #define feather_callback
diff --git a/include/litmus/litmus_proc.h b/include/litmus/litmus_proc.h
index 6800e725d48c..06b73b746f87 100644
--- a/include/litmus/litmus_proc.h
+++ b/include/litmus/litmus_proc.h
@@ -23,3 +23,6 @@ void remove_plugin_proc_dir(struct sched_plugin* plugin);
  * -EFAULT. */
 int copy_and_chomp(char *kbuf, unsigned long ksize,
                    __user const char* ubuf, unsigned long ulength);
+
+int litmus_preempt_and_irq_proc_handler(struct ctl_table *, int,
+                void __user *, size_t *, loff_t *);
diff --git a/litmus/Makefile b/litmus/Makefile
index 61f7c9576d35..62878336f560 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -32,6 +32,7 @@ obj-y     = sched_plugin.o litmus.o \
             sync.o
 
 
+obj-$(CONFIG_EXYNOS_MCT) += clock.o
 obj-$(CONFIG_CPU_V7) += lockdown.o
 obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
 obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
diff --git a/litmus/clock.c b/litmus/clock.c
new file mode 100644
index 000000000000..d0ec082cd496
--- /dev/null
+++ b/litmus/clock.c
@@ -0,0 +1,31 @@
+#include <linux/module.h>
+#include <linux/clocksource.h>
+
+#include <asm/timex.h>
+
+/*
+ * Note that:
+ * - cycle_t =>  linux/clocksource.h
+ * - cycles_t => asm/timex.h (LITMUS^RT)
+ */
+
+extern struct clocksource mct_frc;
+
+static int __init init_litmus_clock(void)
+{
+        BUILD_BUG_ON(sizeof(cycles_t) != sizeof(cycle_t));
+
+        printk("LITMUS^RT: Set up %s as clock source. "
+                        "mult=%u  shift=%u  mask=0x%016llx\n",
+                        mct_frc.name, mct_frc.mult, mct_frc.shift,
+                        mct_frc.mask);
+
+        return 0;
+}
+
+static void exit_litmus_clock(void)
+{
+}
+
+module_init(init_litmus_clock);
+module_exit(exit_litmus_clock);
diff --git a/litmus/color.c b/litmus/color.c
index 278f541e07c1..96ccf3efd228 100644
--- a/litmus/color.c
+++ b/litmus/color.c
@@ -14,6 +14,7 @@
 
 #include <litmus/color.h>
 #include <litmus/litmus.h> /* for in_list(...) */
+#include <litmus/clock.h>
 
 #define PAGES_PER_COLOR 1024
 
@@ -342,8 +343,8 @@ out:
 
 static void __init print_cycle_count(void *_unused)
 {
-        cycles_t cycles = get_cycles();
-        printk("CPU %2d: %10lu cycles\n", smp_processor_id(), cycles);
+        cycles_t cycles = litmus_get_cycles();
+        printk("CPU %2d: %10llu cycles\n", smp_processor_id(), cycles);
 }
 
 static int __init init_color(void)
diff --git a/litmus/color_proc.c b/litmus/color_proc.c
index abfa4c8a3c3b..3c301f059b9f 100644
--- a/litmus/color_proc.c
+++ b/litmus/color_proc.c
@@ -2,6 +2,7 @@
 #include <linux/sysctl.h>
 #include <linux/slab.h>
 
+#include <litmus/litmus_proc.h>
 #include <litmus/sched_trace.h>
 #include <litmus/color.h>
 
@@ -32,6 +33,14 @@ unsigned long color_chunk;
 int cache_preempt;
 int lock_cache;
 
+#ifdef CONFIG_CPU_V7
+static int l1_prefetch_proc;
+static int l2_prefetch_hint_proc;
+static int l2_double_linefill_proc;
+static int l2_data_prefetch_proc;
+static int lockdown_proc;
+#endif
+
 #define INFO_BUFFER_SIZE 100
 static char info_buffer[100];
 
@@ -59,6 +68,49 @@ static struct ctl_table cache_table[] =
                 .data           = &color_cache_info.nr_colors,
                 .maxlen         = sizeof(color_cache_info.nr_colors),
         },
+#ifdef CONFIG_CPU_V7
+        {
+                .procname       = "prefetch_test",
+                .mode           = 0644,
+                .proc_handler   = litmus_test_prefetch_proc_handler,
+        },
+        {
+                .procname       = "l1_prefetch",
+                .mode           = 0644,
+                .proc_handler   = litmus_l1_prefetch_proc_handler,
+                .data           = &l1_prefetch_proc,
+                .maxlen         = sizeof(l1_prefetch_proc),
+        },
+        {
+                .procname       = "l2_prefetch_hint",
+                .mode           = 0644,
+                .proc_handler   = litmus_l2_prefetch_hint_proc_handler,
+                .data           = &l2_prefetch_hint_proc,
+                .maxlen         = sizeof(l2_prefetch_hint_proc),
+        },
+        {
+                .procname       = "l2_double_linefill",
+                .mode           = 0644,
+                .proc_handler   = litmus_l2_double_linefill_proc_handler,
+                .data           = &l2_double_linefill_proc,
+                .maxlen         = sizeof(l2_double_linefill_proc),
+        },
+        {
+                .procname       = "l2_data_prefetch",
+                .mode           = 0644,
+                .proc_handler   = litmus_l2_data_prefetch_proc_handler,
+                .data           = &l2_data_prefetch_proc,
+                .maxlen         = sizeof(l2_data_prefetch_proc),
+        },
+        {
+                .procname       = "lockdown",
+                .mode           = 0644,
+                .proc_handler   = litmus_lockdown_proc_handler,
+                .data           = &lockdown_proc,
+                .maxlen         = sizeof(lockdown_proc),
+
+        },
+#endif
         { }
 };
 
@@ -120,13 +172,6 @@ static struct ctl_table color_table[] =
                 .mode           = 0555,
                 .child          = cache_table,
         },
-#ifdef CONFIG_CPU_V7
-        {
-                .procname       = "prefetch_test",
-                .mode           = 0644,
-                .proc_handler   = litmus_test_prefetch,
-        },
-#endif
         { }
 };
 
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 531a2245133d..4cb6b5c47fd6 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -304,7 +304,7 @@ asmlinkage long sys_null_call(cycles_t __user *ts)
         cycles_t now;
 
         if (ts) {
-                now = get_cycles();
+                now = litmus_get_cycles();
                 ret = put_user(now, ts);
         }
 
@@ -659,7 +659,13 @@ extern struct sched_plugin linux_sched_plugin;
 #if defined(CONFIG_CPU_V7) && !defined(CONFIG_HW_PERF_EVENTS)
 static void __init litmus_enable_perfcounters_v7(void *_ignore)
 {
-        u32 enable_val;
+        u32 enable_val = 0;
+
+        /* disable performance monitoring */
+        asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r" (0x00000006));
+
+        /* disable all events */
+        asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (0xffffffff));
 
         /* write 1 to enable user-mode access to the performance counter */
         asm volatile("mcr p15, 0, %0, c9, c14, 0" : : "r" (1));
@@ -667,13 +673,21 @@ static void __init litmus_enable_perfcounters_v7(void *_ignore)
         /* disable counter overflow interrupts (just in case) */
         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (0x8000000f));
 
-        enable_val = 1;         /* bit 1 enables the counters */
-        enable_val |= 2;        /* resets cycle counter to zero */
+        /* select event zero */
+        asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (0));
+
+        /* count cycles in the selected event zero */
+        asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (0x00000011));
+
+        enable_val |= 1;        /* bit 1 enables the counters */
+        enable_val |= 2;        /* resets event counters to zero */
+        enable_val |= 4;        /* resets cycle counter to zero */
+
         /* performance monitor control register: enable all counters */
         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(enable_val));
 
-        /* enables counters (currently just cycle counter) */
-        asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r"(0x80000000));
+        /* enables counters (cycle counter and event 1) */
+        asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r"(0x80000001));
 }
 
 static void __init litmus_enable_perfcounters(void)
diff --git a/litmus/lockdown.c b/litmus/lockdown.c
index 7cfa24859c0d..ac7b9a24a512 100644
--- a/litmus/lockdown.c
+++ b/litmus/lockdown.c
@@ -4,14 +4,19 @@
 #include <linux/io.h>
 #include <linux/spinlock.h>
 #include <linux/time.h>
+#include <linux/sysctl.h>
+#include <linux/mutex.h>
+#include <linux/math64.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
 
-#include <linux/smp.h> /* smp_call_func */
-#include <asm/processor.h> /* cpu relax */
+#include <litmus/clock.h>
 
 #include <asm/hardware/cache-l2x0.h>
 #include <asm/cacheflush.h>
 
 #include <litmus/color.h>
+#include <litmus/debug_trace.h>
 
 static void __iomem *cache_base;
 static void __iomem *lockreg_d;
@@ -21,6 +26,10 @@ static raw_spinlock_t prefetch_lock;
 static u32 cache_id;
 static int nr_lockregs;
 
+struct mutex actlr_mutex;
+struct mutex l2x0_prefetch_mutex;
+struct mutex lockdown_proc;
+
 #define ld_d_reg(cpu) ({ int __cpu = cpu; \
                         void __iomem *__v = cache_base + L2X0_LOCKDOWN_WAY_D_BASE + \
                         __cpu * L2X0_LOCKDOWN_STRIDE; __v; })
@@ -55,8 +64,8 @@ static u32 read_in_page(u32 lock_val,
 "       .align 5\n"
 "       str     %[lockval], [%[cachereg]]\n"
 "1:     ldr     %[val], [%[addr]], #32          @ 32 bytes = 1 cache line\n"
-"       cmp     %[addr], %[end]\n"
-"       bne     1b\n"
+"       cmp     %[end], %[addr]                 @ subtracts addr from end\n"
+"       bgt     1b\n                            @ read more, if necessary\n"
 "       str     %[unlockval], [%[cachereg]]\n"
         : [addr] "+r" (start),
           [val] "+r" (v)
@@ -105,7 +114,7 @@ static u32 unlocked[MAX_NR_WAYS]  = {
         0xFFFF7FFF,
 };
 
-static void dump_lockdown_registers(void)
+static void print_lockdown_registers(void)
 {
         int i;
 
@@ -117,6 +126,338 @@ static void dump_lockdown_registers(void)
         }
 }
 
+/* Operate on the Cortex-A9's ACTLR register */
+#define ACTLR_L2_PREFETCH_HINT  (1 << 1)
+#define ACTLR_L1_PREFETCH       (1 << 2)
+
+/*
+ * Change the ACTLR.
+ * @mode        - If 1 (0), set (clear) the bit given in @mask in the ACTLR.
+ * @mask        - A mask in which one bit is set to operate on the ACTLR.
+ */
+static void actlr_change(int mode, int mask)
+{
+        u32 orig_value, new_value, reread_value;
+
+        if (0 != mode && 1 != mode) {
+                printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n",
+                                __FUNCTION__);
+                return;
+        }
+
+        /* get the original value */
+        asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (orig_value));
+
+        if (0 == mode)
+                new_value = orig_value & ~(mask);
+        else
+                new_value = orig_value | mask;
+
+        asm volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (new_value));
+        asm volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (reread_value));
+
+        printk("ACTLR: orig: 0x%8x  wanted: 0x%8x  new: 0x%8x\n",
+                        orig_value, new_value, reread_value);
+}
+
+int litmus_l1_prefetch_proc_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int ret, mode;
+
+        mutex_lock(&actlr_mutex);
+        ret = proc_dointvec(table, write, buffer, lenp, ppos);
+
+        if (!ret && write) {
+                mode = *((int*)table->data);
+                actlr_change(mode, ACTLR_L1_PREFETCH);
+        }
+        mutex_unlock(&actlr_mutex);
+
+        return ret;
+}
+
+int litmus_l2_prefetch_hint_proc_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int ret, mode;
+
+        mutex_lock(&actlr_mutex);
+        ret = proc_dointvec(table, write, buffer, lenp, ppos);
+        if (!ret && write) {
+                mode = *((int*)table->data);
+                actlr_change(mode, ACTLR_L2_PREFETCH_HINT);
+        }
+        mutex_unlock(&actlr_mutex);
+
+        return ret;
+}
+
+
+/* Operate on the PL-310's Prefetch Control Register, L2X0_PREFETCH_CTRL */
+#define L2X0_PREFETCH_DOUBLE_LINEFILL   (1 << 30)
+#define L2X0_PREFETCH_DATA_PREFETCH     (1 << 28)
+static void l2x0_prefetch_change(int mode, int mask)
+{
+        u32 orig_value, new_value, reread_value;
+
+        if (0 != mode && 1 != mode) {
+                printk(KERN_WARNING "Called %s with mode != 0 and mode != 1.\n",
+                                __FUNCTION__);
+                return;
+        }
+
+        orig_value = readl_relaxed(cache_base + L2X0_PREFETCH_CTRL);
+
+        if (0 == mode)
+                new_value = orig_value & ~(mask);
+        else
+                new_value = orig_value | mask;
+
+        writel_relaxed(new_value, cache_base + L2X0_PREFETCH_CTRL);
+        reread_value = readl_relaxed(cache_base + L2X0_PREFETCH_CTRL);
+
+        printk("l2x0 prefetch: orig: 0x%8x  wanted: 0x%8x  new: 0x%8x\n",
+                        orig_value, new_value, reread_value);
+}
+
+int litmus_l2_double_linefill_proc_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int ret, mode;
+
+        mutex_lock(&l2x0_prefetch_mutex);
+        ret = proc_dointvec(table, write, buffer, lenp, ppos);
+        if (!ret && write) {
+                mode = *((int*)table->data);
+                l2x0_prefetch_change(mode, L2X0_PREFETCH_DOUBLE_LINEFILL);
+        }
+        mutex_unlock(&l2x0_prefetch_mutex);
+
+        return ret;
+}
+
+int litmus_l2_data_prefetch_proc_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int ret, mode;
+
+        mutex_lock(&l2x0_prefetch_mutex);
+        ret = proc_dointvec(table, write, buffer, lenp, ppos);
+        if (!ret && write) {
+                mode = *((int*)table->data);
+                l2x0_prefetch_change(mode, L2X0_PREFETCH_DATA_PREFETCH);
+        }
+        mutex_unlock(&l2x0_prefetch_mutex);
+
+        return ret;
+}
+
+int litmus_lockdown_proc_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+        int ret, lockdown, *data_ptr;
+        data_ptr = (int*) table->data;
+
+        mutex_lock(&lockdown_proc);
+        if (!write) {
+                lockdown = readl_relaxed(lockreg_d);
+                *data_ptr = lockdown;
+        }
+
+        ret = proc_dointvec(table, write, buffer, lenp, ppos);
+        if (!ret && write) {
+                lockdown = *((int*)table->data);
+                writel_relaxed(lockdown, lockreg_d);
+        }
+        mutex_unlock(&lockdown_proc);
+
+        return ret;
+}
+
+#define TRIALS 1000
+
+static int test_get_cycles_overhead(void)
+{
+        u64 sum = 0, min = (u64)-1, max = 0;
+        unsigned long flags;
+        cycles_t a, b;
+        int i;
+
+        for (i = 0; i < TRIALS; i++) {
+                u64 diff;
+                local_irq_save(flags);
+                preempt_disable();
+                a = litmus_get_cycles();
+                b = litmus_get_cycles();
+                preempt_enable();
+                local_irq_restore(flags);
+                diff = b - a;
+                if (diff > max)
+                        max = diff;
+                if (diff < min)
+                        min = diff;
+                sum += diff;
+        }
+        printk("cycle test: avg: %llu  min: %llu  max: %llu\n",
+                        div64_u64(sum, TRIALS), min, max);
+        return 0;
+}
+
+static long update_timeval(struct timespec lhs, struct timespec rhs)
+{
+        long val;
+        struct timespec ts;
+
+        ts = timespec_sub(rhs, lhs);
+        val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec;
+
+        return val;
+}
+
+/*
+ * 16 *  4 pages to use as colors 0->15 in 4 ways, and
+ * 16 * 32 pages to use as colors 0->15 in 32 ways
+ * Don't change these, because it will break things.
+ */
+#define NR_COLORS       16
+#define CTRL_PAGES      (NR_COLORS * 4)
+#define THRASH_PAGES    (NR_COLORS * 32)
+#define TOTAL_PAGES     (CTRL_PAGES + THRASH_PAGES)
+#define WAY_OFFSET      4
+
+static void thrash(void *vaddr)
+{
+        void *thrash_pages = vaddr + CTRL_PAGES * PAGE_SIZE;
+
+        /* try and flush it */
+        v7_flush_kern_dcache_area(vaddr, CTRL_PAGES * PAGE_SIZE);
+
+        /* thrash. don't lock down, we want to fill the dcache with these */
+        read_in_page(UNLOCK_ALL, UNLOCK_ALL, thrash_pages,
+                        thrash_pages + THRASH_PAGES * PAGE_SIZE);
+}
+
+//#define READ_TRACE(fmt, args...) TRACE("read_trace: " fmt, ##args)
+#define READ_TRACE(fmt, args...) do { } while (0)
+
+static int test_read_in(void)
+{
+        struct page **pages;
+        cycles_t start, stop;
+        unsigned long flags;
+        void *remapped;
+        u32 sum = 0;
+        int ret = 0, i, j;
+
+        pages = (struct page**) kmalloc(TOTAL_PAGES * sizeof(*pages), GFP_KERNEL);
+        if (!pages) {
+                printk("could not allocate pages array.\n");
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        /* Allocate a bunch of pages. */
+        for (i = 0; i < TOTAL_PAGES; i++) {
+                const unsigned long color = i % NR_COLORS;
+
+                pages[i] = get_colored_page(color);
+                if (!pages[i]) {
+                        printk(KERN_WARNING "%s: no pages available.\n", __FUNCTION__);
+                        ret = -ENOMEM;
+                        goto out_free;
+                }
+        }
+
+        /* Put the pages in a contiguous virtual address space. */
+        remapped = vmap(pages, TOTAL_PAGES, VM_MAP, PAGE_KERNEL);
+        if (!remapped) {
+                printk(KERN_WARNING "%s: bad vmap\n", __FUNCTION__);
+                ret = -EINVAL;
+                goto out_free_colors;
+        }
+
+        /* smaller readings, in bytes */
+        for (i = 8; i <= 4096; i += 8) {
+                thrash(remapped);
+
+                READ_TRACE("small test, i:%d\n", i);
+
+                preempt_disable();
+                local_irq_save(flags);
+                start = litmus_get_cycles();
+                read_in_page(unlocked[WAY_OFFSET], UNLOCK_ALL,
+                                remapped, remapped + i);
+                stop = litmus_get_cycles();
+                local_irq_restore(flags);
+                preempt_enable();
+
+                TRACE("wss, nanoseconds: %4d, %lld\n",
+                                i, litmus_cycles_to_ns(stop - start));
+        }
+
+        for (i = 1; i <= CTRL_PAGES; i += 1) {
+                /* i is the number of pages to read in */
+                /* we will read in from page zero to page i (exclusive) */
+
+                READ_TRACE("start on i:%d\n", i);
+
+                thrash(remapped);
+
+                preempt_disable();
+                local_irq_save(flags);
+                start = litmus_get_cycles();
+                for (j = 0; j < i; j += NR_COLORS) {
+                        /* need to chunk the reads into groups of NR_COLORS
+                         * so we can switch ways
+                         */
+                        void *vaddr_start, *vaddr_end;
+                        int read_start = j, read_end = j + NR_COLORS;
+                        int way = WAY_OFFSET + j / NR_COLORS;
+
+                        if (read_end > i)
+                                read_end = i;
+
+                        vaddr_start = remapped + PAGE_SIZE * read_start;
+                        vaddr_end   = remapped + PAGE_SIZE * read_end;
+
+                        read_in_page(unlocked[way], UNLOCK_ALL,
+                                        vaddr_start, vaddr_end);
+
+                        READ_TRACE("i:%d  j:%d  read_start:%d  read_end:%d  way:%d\n",
+                                        i, j, read_start, read_end, way);
+                }
+                stop = litmus_get_cycles();
+                local_irq_restore(flags);
+                preempt_enable();
+
+                TRACE("wss, nanoseconds: %4lu, %lld\n",
+                                PAGE_SIZE * i,
+                                litmus_cycles_to_ns(stop - start));
+        }
+
+#if 0
+        printk("read in %d pages (avg): %llu cycles  %ld (getnstimeofday) sum: %u\n",
+                        NR_PAGES,
+                        div64_u64(stop - start, TRIALS),
+                        update_timeval(before, after) / TRIALS,
+                        sum);
+#endif
+
+        /* Done with these pages */
+        vunmap(remapped);
+
+out_free_colors:
+        for (i = 0; i < TOTAL_PAGES; i++) {
+                put_page(pages[i]);
+                add_page_to_color_list(pages[i]);
+        }
+out_free:
+        kfree(pages);
+out:
+        return 0;
+}
+
 static void test_lockdown(void *ignore)
 {
         int i;
@@ -129,7 +470,7 @@ static void test_lockdown(void *ignore)
         }
 
         printk("Lockdown initial state:\n");
-        dump_lockdown_registers();
+        print_lockdown_registers();
         printk("---\n");
 
         for (i = 0; i < nr_lockregs; i++) {
@@ -137,7 +478,7 @@ static void test_lockdown(void *ignore)
                 writel_relaxed(2, ld_i_reg(i));
         }
         printk("Lockdown all data=1 instr=2:\n");
-        dump_lockdown_registers();
+        print_lockdown_registers();
         printk("---\n");
 
         for (i = 0; i < nr_lockregs; i++) {
@@ -145,7 +486,7 @@ static void test_lockdown(void *ignore)
                 writel_relaxed(((1 << 8) >> i), ld_i_reg(i));
         }
         printk("Lockdown varies:\n");
-        dump_lockdown_registers();
+        print_lockdown_registers();
         printk("---\n");
 
         for (i = 0; i < nr_lockregs; i++) {
@@ -153,7 +494,7 @@ static void test_lockdown(void *ignore)
                 writel_relaxed(UNLOCK_ALL, ld_i_reg(i));
         }
         printk("Lockdown all zero:\n");
-        dump_lockdown_registers();
+        print_lockdown_registers();
 
         /* Checks that the unlocked array is set up correctly. */
         for (i = 0; i < MAX_NR_WAYS; i++) {
@@ -168,18 +509,6 @@ static void test_lockdown(void *ignore)
         printk("End lockdown test.\n");
 }
 
-static long update_timeval(struct timespec lhs, struct timespec rhs)
-{
-        long val;
-        struct timespec ts;
-
-        ts = timespec_sub(rhs, lhs);
-        val = ts.tv_sec*NSEC_PER_SEC + ts.tv_nsec;
-
-        return val;
-}
-
-#define TRIALS 1000
 static int perf_test(void) {
         struct timespec before, after;
         struct page *page;
@@ -237,8 +566,8 @@ static int perf_test(void) {
 }
 
 #define LOCKREG_TEST_VAL        0x00000002
-int litmus_test_prefetch(struct ctl_table *table, int write, void __user *buffer,
-                size_t *lenp, loff_t *ppos)
+int litmus_test_prefetch_proc_handler(struct ctl_table *table, int write,
+                void __user *buffer, size_t *lenp, loff_t *ppos)
 {
         struct page *page;
         void *vaddr;
@@ -277,7 +606,8 @@ int litmus_test_prefetch(struct ctl_table *table, int write, void __user *buffer
 
         free_page((unsigned long)vaddr);
 
-        perf_test();
+        //test_get_cycles_overhead();
+        test_read_in();
 
         return 0;
 }
@@ -298,6 +628,9 @@ void litmus_setup_lockdown(void __iomem *base, u32 id)
         }
 
         raw_spin_lock_init(&prefetch_lock);
+        mutex_init(&actlr_mutex);
+        mutex_init(&l2x0_prefetch_mutex);
+        mutex_init(&lockdown_proc);
 
         test_lockdown(NULL);
 }